framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,beam_width,attn_dtype,kv_cache_dtype,step,latency
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,1,0.011957333733638128
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,31,0.012042666474978128
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,fp8,1,0.01314666618903478
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,fp8,31,0.012847999731699625
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,63,0.011861333002646765
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,3,0.011952000359694162
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,fp8,63,0.012719999998807907
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,127,0.012133333832025528
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,fp8,127,0.014858666807413101
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,255,0.013493333011865616
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,fp8,255,0.01498666654030482
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,511,0.015471999843915304
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,fp8,511,0.01756799966096878
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,1023,0.020026666422684986
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,fp8,1023,0.0223786657055219
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,2047,0.028730665644009907
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,fp8,2047,0.0324799989660581
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,4095,0.048063998421033226
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,fp8,3,0.013237333546082178
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,fp8,4095,0.05272533496220907
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,1,0.012165332833925882
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,fp8,1,0.013023999830087027
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,3,0.012426666915416718
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,fp8,3,0.012986666212479273
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,7,0.011855999628702799
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,fp8,7,0.012837332983811697
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,15,0.01250133290886879
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,fp8,15,0.013034666577974955
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,31,0.012005332857370377
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,fp8,31,0.012970666090647379
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,63,0.01221866657336553
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,fp8,63,0.013183999806642532
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,127,0.011946666985750198
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,fp8,127,0.014485333114862442
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,255,0.014080000420411428
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,fp8,255,0.015253332753976187
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,511,0.01586666703224182
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,fp8,511,0.017717332889636356
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,1023,0.020282667130231857
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,fp8,1023,0.02256533255179723
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,2047,0.03346666693687439
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,fp8,2047,0.03309866786003113
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,4095,0.04980800052483877
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,fp8,4095,0.05745066702365875
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,1,0.012362666428089142
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,3,0.01246400053302447
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,fp8,1,0.013717333475748697
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,fp8,3,0.013733333597580591
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,7,0.012453333785136541
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,fp8,7,0.01368533323208491
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,15,0.012543999900420507
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,fp8,15,0.013823999712864557
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,31,0.012378666549921036
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,fp8,31,0.013647999614477158
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,63,0.01259200026591619
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,fp8,63,0.013839999834696451
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,fp8,127,0.015210667004187902
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,127,0.012719999998807907
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,255,0.014389333625634512
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,fp8,255,0.01594666639963786
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,511,0.016677333662907284
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,fp8,511,0.01855466639002164
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,2047,0.043807998299598694
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,fp8,2047,0.037962667644023895
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,fp8,1023,0.02380799998839696
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,fp8,7,0.013023999830087027
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,1023,0.02789866675933202
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,fp8,4095,0.05870933334032694
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,15,0.011866666376590729
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,4095,0.07377600173155467
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,1,0.016602666427691776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,fp8,1,0.01743999992807706
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,3,0.016597333053747814
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,fp8,3,0.0173333336909612
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,7,0.01210133358836174
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,7,0.016501333564519882
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,fp8,7,0.017429333180189133
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,15,0.016549333930015564
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,fp8,15,0.01752000053723653
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,31,0.016762666404247284
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,fp8,31,0.017370666066805523
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,fp8,15,0.012885333349307379
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,fp8,63,0.01762666677435239
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,63,0.01669866715868314
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,fp8,127,0.02051199972629547
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,127,0.01717866708834966
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,255,0.020080000162124634
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,fp8,255,0.021722666919231415
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,511,0.031082667410373688
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,1023,0.04643199841181437
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,fp8,1023,0.040549332896868386
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,fp8,511,0.02757333219051361
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,2047,0.07749866445859273
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,fp8,2047,0.06136000156402588
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,4095,0.13531733552614847
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,fp8,4095,0.10094400246938069
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,7,0.011215999722480774
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,1,0.011029332876205444
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,1,0.011850666254758835
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,3,0.01109333336353302
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,3,0.012346666306257248
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,7,0.012367999802033106
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,15,0.011231999844312668
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,15,0.012351999680201212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,31,0.011221333096424738
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,31,0.01227733368674914
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,255,0.012538666526476542
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,63,0.011183999478816986
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,63,0.012613333761692047
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,127,0.011077333241701126
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,127,0.013679999858140945
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,255,0.01381333296497663
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,511,0.012682666381200155
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,511,0.013797332843144735
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,1023,0.012975999464591345
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,1023,0.014501333236694336
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,2047,0.014101333916187286
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,2047,0.015610666324694952
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,4095,0.015706667055686314
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,16383,0.022389332453409832
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,4095,0.01637866720557213
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,8191,0.017952000101407368
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,8191,0.019487999379634857
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,16383,0.024874667326609295
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,1,0.011370666325092316
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,1,0.012047999848922094
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,3,0.011178666104873022
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,3,0.012453333785136541
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,7,0.011136000355084738
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,7,0.012256000190973282
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,15,0.011146667102972666
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,15,0.012576000144084295
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,31,0.01119999960064888
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,31,0.012362666428089142
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,63,0.01101333275437355
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,63,0.012181332955757776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,127,0.011152000476916632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,127,0.013370666652917862
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,255,0.012245333443085352
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,255,0.013663999736309052
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,511,0.012666666259368261
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,511,0.01392000044385592
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,1023,0.012821332861979803
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,1023,0.014752000570297241
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,2047,0.014730667074521383
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,2047,0.016085332880417507
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,4095,0.01618133361140887
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,4095,0.017093333105246227
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,8191,0.01834133391578992
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,8191,0.020517333100239437
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,16383,0.02288000037272771
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,16383,0.025114665428797405
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,1,0.011333333949247995
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,1,0.01249066616098086
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,3,0.011328000575304031
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,3,0.012357333054145178
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,7,0.011237333218256632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,7,0.01240533341964086
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,15,0.011509332805871964
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,15,0.01227733368674914
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,31,0.01137599969903628
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,31,0.012383999923865
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,63,0.011296000331640244
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,63,0.012346666306257248
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,127,0.011370666325092316
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,127,0.013424000392357508
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,255,0.012373333175977072
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,255,0.013552000125249227
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,511,0.013034666577974955
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,511,0.014565333724021912
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,1023,0.013834666460752487
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,1023,0.014901333798964819
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,2047,0.015370666980743408
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,2047,0.01716800034046173
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,4095,0.016282666474580765
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,4095,0.01793066660563151
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,8191,0.018645333747069042
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,8191,0.020202666521072388
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,16383,0.02314666658639908
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,16383,0.025983999172846477
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,1,0.011461333682139715
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,1,0.012181332955757776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,3,0.011343999455372492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,3,0.012026666353146235
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,7,0.011391999820868174
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,7,0.01231466606259346
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,15,0.011584000041087469
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,15,0.012213333199421564
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,31,0.011567999919255575
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,31,0.012426666915416718
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,63,0.011391999820868174
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,63,0.012202666451533636
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,127,0.011530666301647821
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,127,0.013199999928474426
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,1023,0.01443733274936676
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,255,0.013013333082199097
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,255,0.013957332819700241
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,511,0.013834666460752487
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,511,0.01421333352724711
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,1023,0.015247999380032221
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,2047,0.015520000209410986
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,2047,0.016879999389251072
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,16383,0.03046400099992752
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,4095,0.016602666427691776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,4095,0.01770666614174843
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,8191,0.019109333554903667
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,1,0.01232533281048139
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,8191,0.020282667130231857
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,16383,0.026682667434215546
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,1,0.010901333143313726
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,3,0.011120000233252844
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,3,0.012245333443085352
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,7,0.011253333340088526
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,7,0.01267733300725619
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,15,0.011141333729028702
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,15,0.012223999947309494
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,31,0.011146667102972666
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,31,0.012234666695197424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,63,0.010965333630641302
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,63,0.01227733368674914
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,127,0.011461333682139715
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,127,0.013712000101804733
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,255,0.012746666868527731
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,255,0.014005333185195923
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,511,0.013072000195582708
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,511,0.01421333352724711
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,4095,0.01801066721479098
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,1023,0.014127999544143677
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,1023,0.015295999745527903
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,2047,0.015450666348139444
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,2047,0.0162773331006368
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,4095,0.019013332823912304
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,8191,0.020869334538777668
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,8191,0.023957334458827972
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,16383,0.029525332152843475
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,16383,0.03450666616360346
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,1,0.011306667079528173
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,1,0.012165332833925882
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,3,0.011247999966144562
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,3,0.012448000411192576
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,31,0.011274666835864386
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,7,0.011450666934251785
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,7,0.012229333321253458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,15,0.011418666690587997
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,15,0.012495999534924826
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,31,0.012378666549921036
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,255,0.012538666526476542
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,63,0.01118933285276095
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,63,0.012346666306257248
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,127,0.011424000064531961
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,127,0.01369599997997284
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,255,0.013679999858140945
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,511,0.012719999998807907
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,511,0.014970666418472925
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,1023,0.014127999544143677
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,1023,0.015749332805474598
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,2047,0.016522667060295742
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,2047,0.016789333273967106
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,4095,0.018288000176350277
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,4095,0.019637333850065868
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,8191,0.021642667551835377
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,8191,0.02402133246262868
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,16383,0.03070399910211563
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,16383,0.0341386670867602
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,1,0.011530666301647821
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,1,0.012181332955757776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,3,0.011493333925803503
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,3,0.01231466606259346
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,7,0.011535999675591787
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,7,0.01228800043463707
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,15,0.011242666592200598
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,15,0.012229333321253458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,31,0.01163200040658315
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,31,0.012191999703645706
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,63,0.01156266654531161
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,63,0.012351999680201212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,127,0.011493333925803503
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,127,0.013610667238632837
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,255,0.012928000340859095
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,255,0.014245333770910898
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,511,0.013717333475748697
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,511,0.014549333602190018
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,1023,0.014997333288192749
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,1023,0.016352000335852306
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,2047,0.016421332955360413
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,2047,0.017317333569129307
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,4095,0.01842133328318596
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,4095,0.01987733319401741
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,8191,0.022304000953833263
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,8191,0.024570666253566742
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,16383,0.03427733232577642
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,16383,0.036661334335803986
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,1,0.011749333391586939
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,1,0.012362666428089142
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,3,0.011877333124478659
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,3,0.012245333443085352
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,7,0.01180800050497055
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,7,0.012159999459981918
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,15,0.011813333878914515
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,15,0.012442667037248611
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,31,0.01198400060335795
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,31,0.012613333761692047
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,63,0.011781333635250727
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,63,0.012426666915416718
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,127,0.011962667107582092
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,127,0.014074667046467463
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,255,0.014010666559139887
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,255,0.014165333161751429
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,511,0.013834666460752487
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,511,0.015002666662136713
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,1023,0.01552533358335495
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,1023,0.015840000162522
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,2047,0.016938666502634685
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,2047,0.017008000363906223
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,4095,0.02015999952952067
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,4095,0.019685332973798115
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,8191,0.02957333376010259
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,8191,0.02442666639884313
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,16383,0.04561600089073181
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,16383,0.03841600070397059
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,1,0.01309866706530253
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,3,0.013162666310866674
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,fp8,1,0.014463999619086584
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,fp8,3,0.014218666901191076
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,7,0.013007999708255133
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,fp8,7,0.014357333381970724
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,15,0.013157332936922709
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,fp8,15,0.01434133326013883
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,31,0.01321600005030632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,fp8,31,0.014346666634082794
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,fp8,63,0.014373333503802618
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,63,0.013167999684810638
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,127,0.013365333278973898
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,fp8,127,0.015775999675194424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,255,0.014970666418472925
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,fp8,255,0.01623999948302905
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,511,0.01685333376129468
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,fp8,511,0.01918399954835574
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,1023,0.021989333132902782
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,fp8,1023,0.02372266600529353
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,2047,0.03765333443880081
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,fp8,2047,0.03461333364248276
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,1,0.013253333667914072
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,fp8,1,0.01452800010641416
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,3,0.01339200014869372
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,fp8,3,0.014741333822409311
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,7,0.01351999988158544
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,fp8,7,0.014549333602190018
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,15,0.013178666432698568
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,fp8,15,0.014746667196353277
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,fp8,63,0.014848000059525171
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,31,0.013386666774749756
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,fp8,31,0.014538666854302088
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,63,0.013482666263977686
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,127,0.013872000078360239
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,fp8,127,0.01591466615597407
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,255,0.015423999478419622
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,fp8,255,0.016714667280515034
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,511,0.017594666530688603
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,fp8,511,0.019343999524911244
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,1023,0.029653333127498627
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,fp8,1023,0.024666666984558105
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,2047,0.0454720010360082
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,1,0.017093333105246227
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,fp8,2047,0.03879466652870178
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,fp8,1,0.018624000251293182
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,3,0.01711999997496605
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,fp8,3,0.018464000274737675
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,7,0.017242666333913803
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,fp8,7,0.01870399961868922
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,fp8,15,0.018464000274737675
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,15,0.017258666455745697
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,fp8,31,0.018543999642133713
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,31,0.017258666455745697
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,63,0.01718933383623759
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,fp8,63,0.018485333770513535
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,127,0.017642666896184284
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,fp8,127,0.021370666722456615
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,255,0.021013334393501282
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,fp8,255,0.022634667654832203
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,511,0.032101333141326904
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,fp8,1023,0.042821332812309265
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,fp8,511,0.029338667790095013
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,1023,0.047775998711586
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,2047,0.07690666615962982
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,fp8,2047,0.0642986645301183
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,1,0.024005333582560223
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,fp8,1,0.025258667767047882
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,3,0.024010665714740753
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,fp8,3,0.025279998779296875
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,7,0.024149333437283833
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,fp8,7,0.02534399926662445
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,15,0.024080000817775726
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,fp8,15,0.025248001019159954
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,31,0.02405333270629247
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,fp8,31,0.025386666258176167
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,63,0.024341332415739696
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,fp8,63,0.025498665869235992
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,127,0.025221332907676697
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,fp8,127,0.03012266755104065
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,255,0.03701333453257879
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,fp8,255,0.03417599946260452
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,511,0.0528106689453125
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,fp8,1023,0.0684799998998642
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,1023,0.08094933132330577
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,fp8,511,0.04795200129350027
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,2047,0.1393066644668579
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,3,0.010869332899649939
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,1,0.010608000059922537
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,fp8,2047,0.10989333192507426
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,3,0.011781333635250727
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,15,0.010735999792814255
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,7,0.011749333391586939
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,15,0.011770666887362799
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,63,0.01190399999419848
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,31,0.011215999722480774
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,63,0.01081066702802976
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,31,0.011871999750534693
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,127,0.0106133334338665
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,255,0.012357333054145178
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,1,0.011701333026091257
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,255,0.013631999492645264
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,127,0.013349333157142004
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,511,0.014618666221698126
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,511,0.013530666629473368
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,1023,0.015685333559910457
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,1023,0.014864000181357065
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,2047,0.01621333385507266
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,2047,0.018330667167901993
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,4095,0.020586666961510975
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,8191,0.029098667204380035
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,16383,0.04711466530958811
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,7,0.010735999792814255
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,16383,0.05188799897829691
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,8191,0.03346133232116699
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,4095,0.023029332359631855
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,1,0.01090666651725769
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,3,0.010837333897749582
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,1,0.011887999872366587
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,7,0.01097600037852923
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,3,0.011839999506870905
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,7,0.012341332932313284
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,15,0.011120000233252844
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,15,0.011706666400035223
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,31,0.010794666906197866
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,31,0.01198400060335795
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,63,0.011130666981140772
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,127,0.0107893335322539
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,63,0.012159999459981918
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,127,0.013482666263977686
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,255,0.012586666891972223
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,255,0.014069333672523499
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,511,0.013514666507641474
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,511,0.015530666957298914
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,1023,0.01532799998919169
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,1023,0.01621333385507266
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,2047,0.016778666526079178
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,2047,0.018981333822011948
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,4095,0.020810666183630627
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,4095,0.023546665906906128
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,8191,0.02978666623433431
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,8191,0.033413333197434746
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,16383,0.049498667319615684
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,16383,0.05346666773160299
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,1,0.011477333803971609
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,1,0.012357333054145178
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,3,0.011328000575304031
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,3,0.012272000312805176
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,7,0.01118933285276095
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,7,0.01198400060335795
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,15,0.011482667177915573
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,15,0.012341332932313284
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,31,0.01166933278242747
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,31,0.012432000289360682
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,63,0.011418666690587997
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,63,0.012309333930412928
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,127,0.01128000020980835
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,127,0.013823999712864557
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,255,0.013386666774749756
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,255,0.014394666999578476
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,511,0.014538666854302088
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,511,0.016389333953460056
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,1023,0.01562133307258288
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,1023,0.01632533346613248
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,2047,0.01695466662446658
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,2047,0.019199999670187633
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,4095,0.022074667116006214
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,4095,0.023562667270501454
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,8191,0.03807466725508372
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,8191,0.03495999922355016
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,16383,0.05472533404827118
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,16383,0.05852800110975901
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,1,0.01221866657336553
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,1,0.012479999413092932
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,3,0.01210133358836174
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,3,0.012560000022252401
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,7,0.012106666962305704
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,7,0.013258667041858038
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,15,0.011871999750534693
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,15,0.012538666526476542
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,31,0.012005332857370377
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,31,0.012357333054145178
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,63,0.0122079998254776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,63,0.012538666526476542
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,127,0.012719999998807907
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,127,0.014282666146755219
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,255,0.0138026662170887
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,255,0.014554666976133982
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,511,0.014597332725922266
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,511,0.016058667252461117
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,1023,0.016271999726692837
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,1023,0.016693333784739178
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,2047,0.017477333545684814
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,2047,0.01904533306757609
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,4095,0.02972800036271413
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,4095,0.02378133436044057
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,8191,0.04478399952252706
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,8191,0.03935466706752777
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,16383,0.07658133407433827
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,16383,0.060720001657803856
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,1,0.01588800052801768
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,fp8,1,0.016970666746298473
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,3,0.015541333705186844
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,fp8,3,0.017050666113694508
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,7,0.015781333049138386
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,15,0.015872000406185787
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,fp8,7,0.017194667210181553
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,fp8,15,0.017055999487638474
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,31,0.015722667177518208
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,fp8,31,0.01687466725707054
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,63,0.015802666544914246
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,fp8,63,0.0170666662355264
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,127,0.01617066686352094
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,fp8,127,0.0183999997874101
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,255,0.01775466650724411
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,fp8,255,0.019093333433071773
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,511,0.01988799994190534
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,fp8,511,0.021856000026067097
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,1023,0.03258133431275686
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,fp8,1023,0.02865600089232127
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,1,0.01903466631968816
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,fp8,1,0.020666666328907013
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,fp8,3,0.020400000115235645
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,fp8,7,0.020506666352351505
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,3,0.019296000401178997
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,7,0.01886933296918869
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,15,0.019039999693632126
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,fp8,15,0.020714666694402695
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,31,0.019071999937295914
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,fp8,31,0.02075200031201045
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,63,0.019178666174411774
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,fp8,63,0.02049066623051961
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,127,0.019461333751678467
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,fp8,127,0.023120000958442688
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,255,0.023365333676338196
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,fp8,255,0.024864000578721363
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,511,0.035274667044480644
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,1023,0.05136533578236898
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,fp8,511,0.03128000100453695
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,1,0.025813333690166473
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,fp8,1023,0.04562133550643921
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,fp8,1,0.02770666778087616
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,3,0.025968000292778015
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,fp8,3,0.02812266598145167
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,7,0.025936000049114227
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,fp8,7,0.02787200113137563
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,15,0.02611200014750163
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,31,0.026149332523345947
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,fp8,15,0.027962667246659596
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,fp8,31,0.02771199991305669
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,63,0.02619733413060506
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,fp8,63,0.02826133370399475
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,127,0.02812800059715907
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,fp8,127,0.03235200047492981
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,255,0.04012266546487808
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,fp8,255,0.038245332737763725
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,511,0.05436266462008158
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,fp8,511,0.05206400156021118
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,1,0.03868799904982249
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,fp8,1,0.04043200115362803
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,1023,0.08425600330034892
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,3,0.038431999584039055
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,fp8,1023,0.07313066720962524
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,fp8,3,0.04055466751257578
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,7,0.03852800031503042
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,15,0.038746667404969536
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,fp8,7,0.040607998768488564
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,fp8,15,0.040618665516376495
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,fp8,31,0.04074133435885111
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,31,0.03881066789229711
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,fp8,63,0.040522667268911995
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,127,0.04595733185609182
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,fp8,127,0.05288533369700114
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,63,0.03982933362325033
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,255,0.05938133100668589
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,1023,0.14724799990653992
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,fp8,255,0.0617386649052302
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,fp8,1023,0.12334932883580525
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,511,0.08890666564305623
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,1,0.024362665911515553
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,fp8,511,0.084906667470932
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,fp8,1,0.026149332523345947
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,3,0.02426133304834366
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,fp8,3,0.02587199956178665
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,fp8,7,0.02606400102376938
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,7,0.024405332903067272
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,31,0.0245919997493426
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,fp8,31,0.025946666797002155
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,15,0.02465066562096278
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,63,0.024522667129834492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,fp8,63,0.026186667382717133
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,127,0.025349333882331848
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,fp8,15,0.026074667771657307
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,fp8,127,0.02846933404604594
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,255,0.03019733230272929
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,fp8,255,0.03068800022204717
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,1,0.030693332354227703
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,fp8,1,0.03268266717592875
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,3,0.030853333572546642
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,fp8,3,0.03249066571394602
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,7,0.031008000175158184
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,fp8,7,0.03249600032965342
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,15,0.031162666777769726
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,fp8,15,0.03268799930810928
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,31,0.031141333281993866
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,fp8,31,0.032655999064445496
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,63,0.031231999397277832
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,fp8,63,0.03303466737270355
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,127,0.033376000821590424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,fp8,127,0.03765333443880081
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,1,0.04385066529115041
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,255,0.04604800045490265
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,fp8,1,0.04673066735267639
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,fp8,255,0.04200000067551931
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,3,0.04382933179537455
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,7,0.043882668018341064
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,fp8,3,0.04756266872088114
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,fp8,7,0.046757335464159645
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,15,0.04423466821511587
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,31,0.044346665342648826
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,fp8,15,0.04772266745567322
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,63,0.04640533526738485
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,fp8,31,0.04701866706212362
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,fp8,63,0.04692266881465912
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,127,0.05133866767088572
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,fp8,127,0.05690133571624756
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,fp8,255,0.06899199883143108
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,255,0.06568533182144165
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,1,0.06850666801134746
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,fp8,1,0.07225066423416138
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,3,0.0689279983441035
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,fp8,3,0.07211199899514516
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,7,0.06885866820812225
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,fp8,7,0.07191999753316243
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,15,0.06924266616503398
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,fp8,15,0.07232533395290375
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,fp8,31,0.07252266506354015
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,31,0.07123200098673503
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,63,0.07517866790294647
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,fp8,63,0.07399466633796692
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,127,0.07880533238252004
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,fp8,127,0.09402133027712505
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,255,0.10337600111961365
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,fp8,255,0.10898133118947347
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,1,0.04784533381462097
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,fp8,1,0.04982399940490723
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,3,0.04794666667779287
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,fp8,3,0.04944533109664917
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,7,0.048122664292653404
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,fp8,7,0.04970133304595947
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,31,0.04811733464399973
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,fp8,31,0.04975466430187225
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,fp8,63,0.04962133367856344
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,15,0.048170665899912514
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,63,0.048613334695498146
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,fp8,15,0.04969066878159841
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,127,0.0537066658337911
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,fp8,127,0.054133335749308266
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,1,0.059658666451772056
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,fp8,1,0.06250133117039998
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,3,0.05955199897289276
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,fp8,3,0.06238933404286703
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,fp8,7,0.06271466612815857
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,7,0.059546664357185364
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,15,0.06027733286221822
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,fp8,15,0.06241066753864288
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,31,0.06069866816202799
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,fp8,31,0.06262399752934773
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,63,0.0651146670182546
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,fp8,63,0.06264000137646993
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,127,0.07010133564472198
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,fp8,127,0.07523199915885925
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,1,0.08661333719889323
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,fp8,1,0.09116799632708232
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,3,0.08622399965922038
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,fp8,3,0.09143466750780742
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,7,0.08611733714739482
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,fp8,7,0.09166933099428813
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,15,0.08803199728329976
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,fp8,15,0.09167466560999553
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,31,0.09180266658465068
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,fp8,31,0.0922986666361491
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,63,0.0941439966360728
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,fp8,63,0.09640533725420634
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,127,0.09710400303204854
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,fp8,127,0.11373333136240642
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,1,0.13523200154304504
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,fp8,1,0.1427893340587616
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,fp8,3,0.14147200187047324
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,3,0.13424533605575562
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,7,0.13665599624315897
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,fp8,15,0.1432799994945526
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,fp8,7,0.1418719987074534
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,15,0.14030399918556213
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,fp8,31,0.14593600233395895
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,31,0.14198399583498636
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,1,0.012282667060693106
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,63,0.14453333616256714
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,1,0.011109333485364914
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,7,0.011231999844312668
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,3,0.012506666282812754
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,fp8,63,0.14621333281199136
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,15,0.011136000355084738
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,3,0.011120000233252844
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,15,0.012373333175977072
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,31,0.01118933285276095
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,7,0.012037333101034164
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,63,0.012213333199421564
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,63,0.011029332876205444
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,127,0.01129066695769628
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,31,0.01201066623131434
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,127,0.013359999905029932
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,255,0.012186666329701742
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,255,0.014602666099866232
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,127,0.15286399920781454
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,511,0.014101333916187286
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,511,0.01594666639963786
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,1023,0.01616000011563301
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,1023,0.01850133389234543
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,fp8,127,0.18071999152501425
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,2047,0.02027733375628789
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,2047,0.023029332359631855
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,4095,0.03288000077009201
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,8191,0.04798933366934458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,4095,0.02935466667016347
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,8191,0.05229333539803823
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,16383,0.08378133177757263
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,1,0.011674666156371435
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,1,0.012149333953857422
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,3,0.011541333049535751
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,16383,0.09291199843088786
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,3,0.012714666624863943
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,7,0.01137599969903628
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,7,0.012229333321253458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,15,0.01157333329319954
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,15,0.012367999802033106
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,31,0.011493333925803503
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,31,0.012362666428089142
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,63,0.01126933346192042
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,63,0.012549333274364471
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,127,0.01137599969903628
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,127,0.013258667041858038
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,255,0.012789333860079447
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,255,0.016037333756685257
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,511,0.014469332993030548
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,511,0.016544000556071598
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,1023,0.016554666062196095
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,1023,0.01854933301607768
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,2047,0.020869334538777668
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,2047,0.02346133440732956
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,4095,0.02977066735426585
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,4095,0.03329599897066752
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,8191,0.04937600096066793
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,8191,0.05433600147565206
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,16383,0.0846613347530365
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,16383,0.10105066498120625
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,1,0.011648000528415045
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,1,0.012869333227475485
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,3,0.011776000261306763
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,3,0.012944000462690989
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,7,0.012085333466529846
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,7,0.01328533391157786
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,15,0.011994666109482447
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,15,0.012810666114091873
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,31,0.011989332735538483
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,31,0.012896000097195307
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,63,0.011920000116030375
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,63,0.012837332983811697
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,127,0.012053333222866058
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,511,0.016271999726692837
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,127,0.014826666563749313
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,255,0.012970666090647379
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,255,0.016293333222468693
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,511,0.015301333119471868
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,1023,0.017082666357358296
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,1023,0.01913600042462349
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,2047,0.020954666038354237
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,2047,0.023823998868465424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,4095,0.033887999753157295
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,4095,0.03469866762558619
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,8191,0.06187200049559275
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,8191,0.05869866907596588
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,16383,0.08768000205357869
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,16383,0.10013866424560547
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,1,0.0120319997270902
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,1,0.013045333325862885
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,3,0.012175999581813812
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,15,0.012335999558369318
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,3,0.013264000415802002
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,7,0.012250666817029318
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,7,0.01331199953953425
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,15,0.013221333424250284
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,31,0.011898666620254517
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,31,0.012794667234023413
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,63,0.012213333199421564
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,255,0.014458666245142618
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,63,0.013440000514189402
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,127,0.012506666282812754
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,127,0.014794666320085526
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,255,0.016000000139077503
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,511,0.015439999600251516
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,511,0.017674667139848072
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,1023,0.017386666188637417
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,1023,0.019061333189407986
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,2047,0.030368000268936157
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,2047,0.02513599892457326
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,4095,0.04562133550643921
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,4095,0.037477334340413414
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,8191,0.07600533465544383
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,8191,0.06083733340104421
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,16383,0.13622400164604187
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,16383,0.10482133428255717
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,1,0.03998400022586187
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,fp8,1,0.039861333866914116
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,3,0.0409706657131513
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,fp8,7,0.039664000272750854
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,7,0.04091733445723852
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,fp8,3,0.0393653338154157
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,fp8,15,0.039936001102129616
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,15,0.04460266729195913
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,fp8,31,0.040565334260463715
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,31,0.046485334634780884
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,63,0.04828266799449921
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,fp8,63,0.04362666606903076
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,1,0.06666666766007741
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,fp8,1,0.06363733112812042
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,3,0.06822933256626129
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,fp8,3,0.06419733166694641
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,7,0.07127999762694041
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,15,0.07368533313274384
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,fp8,7,0.06493333478768666
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,fp8,15,0.06565333406130473
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,31,0.0745066652695338
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,fp8,31,0.06804800033569336
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,63,0.075573335091273
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,fp8,63,0.07445866862932841
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,1,0.11986133456230164
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,fp8,1,0.11516799529393514
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,fp8,3,0.11543466647466023
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,3,0.12433066964149475
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,7,0.12758933504422507
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,fp8,7,0.1148426632086436
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,15,0.130431999762853
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,fp8,15,0.12164800365765889
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,31,0.12957333525021872
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,fp8,31,0.12782933314641318
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,1,0.2053920030593872
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,63,0.1311893363793691
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,fp8,63,0.12827733159065247
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,fp8,1,0.204367995262146
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,3,0.20682666699091592
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,fp8,3,0.20595733324686685
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,7,0.2108373244603475
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,fp8,7,0.2073919971783956
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,fp8,15,0.21451733509699503
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,15,0.21883734067281088
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,31,0.2230293353398641
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,fp8,31,0.21663467089335123
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,63,0.22924266258875528
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,fp8,63,0.22246400515238443
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,fp8,1,0.07877866427103679
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,float16,1,0.07868800063927968
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,float16,3,0.07784000039100647
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,fp8,3,0.077824001510938
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,float16,7,0.0787306676308314
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,fp8,7,0.07835199932257335
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,float16,15,0.07814933359622955
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,fp8,15,0.07895466685295105
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,float16,31,0.0790719985961914
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,fp8,31,0.07960533102353413
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,float16,1,0.13316266735394797
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,fp8,1,0.12587199608484903
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,float16,3,0.13406933347384134
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,float16,7,0.1343786617120107
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,fp8,3,0.12662399808565775
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,fp8,7,0.12753599882125854
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,float16,15,0.13446399569511414
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,fp8,15,0.12795199950536093
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,fp8,31,0.1297546625137329
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,float16,31,0.13429333766301474
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,float16,1,0.24639999866485596
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,fp8,1,0.23300800720850626
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,float16,3,0.24318399031956991
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,fp8,3,0.23293334245681763
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,float16,7,0.24522133668263754
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,fp8,7,0.233130673567454
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,float16,15,0.2458666761716207
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,fp8,15,0.23625600337982178
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,float16,31,0.24786667029062906
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,float16,1,0.40672532717386883
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,fp8,31,0.23696533838907877
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,fp8,1,0.40567998091379803
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,float16,3,0.4043946663538615
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,fp8,3,0.4081813494364421
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,1,0.011338666081428528
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,fp8,1,0.012181332955757776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,float16,7,0.4135040044784546
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,3,0.011253333340088526
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,fp8,3,0.01239466667175293
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,fp8,7,0.012026666353146235
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,15,0.011530666301647821
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,7,0.011306667079528173
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,fp8,7,0.4129759867986043
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,fp8,15,0.012357333054145178
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,float16,15,0.42377599080403644
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,fp8,31,0.01227733368674914
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,31,0.011354666203260422
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,fp8,63,0.012266666938861212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,63,0.011695999652147293
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,127,0.011418666690587997
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,255,0.012746666868527731
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,fp8,255,0.014271999398867289
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,fp8,127,0.013477332890033722
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,fp8,15,0.4150773286819458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,511,0.014906667172908783
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,fp8,511,0.017077332983414333
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,1023,0.018954666952292126
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,float16,31,0.42843198776245117
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,fp8,1023,0.021738665799299877
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,2047,0.027674667537212372
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,fp8,2047,0.03194666653871536
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,fp8,4095,0.05115200082461039
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,4095,0.04530133306980133
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,8191,0.08213866750399272
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,fp8,8191,0.09425066908200581
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,fp8,1,0.012847999731699625
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,3,0.012026666353146235
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,1,0.012117333710193634
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,fp8,31,0.4198880195617676
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,fp8,3,0.013013333082199097
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,7,0.011861333002646765
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,fp8,7,0.012863999853531519
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,15,0.011898666620254517
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,fp8,15,0.013114667187134424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,31,0.011936000237862269
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,fp8,31,0.012794667234023413
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,63,0.011813333878914515
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,127,0.012037333101034164
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,fp8,127,0.014314666390419006
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,fp8,63,0.012906666845083237
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,255,0.013776000589132309
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,fp8,255,0.015450666348139444
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,511,0.01552533358335495
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,fp8,511,0.017914666483799618
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,1023,0.019813333948453266
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,fp8,1023,0.022367998957633972
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,2047,0.028704000016053517
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,fp8,2047,0.03241066634654999
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,4095,0.04816000163555145
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,fp8,4095,0.05235200126965841
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,8191,0.08478933572769165
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,fp8,8191,0.09883733590443929
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,1,0.012191999703645706
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,fp8,1,0.01349866638580958
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,3,0.011674666156371435
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,fp8,3,0.013162666310866674
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,7,0.012106666962305704
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,fp8,7,0.013343999783198038
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,15,0.01209066684047381
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,fp8,15,0.013189333180586496
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,31,0.01231466606259346
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,fp8,31,0.013471999516089758
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,63,0.011941333611806234
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,fp8,255,0.01534933348496755
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,fp8,63,0.013151999562978745
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,127,0.012469333906968435
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,fp8,127,0.014965333044528961
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,255,0.01414399966597557
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,511,0.015360000232855478
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,fp8,511,0.017925333231687546
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,1023,0.01966933285196622
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,fp8,1023,0.022448000808556873
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,2047,0.0324799989660581
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,fp8,2047,0.03293866664171219
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,4095,0.05287466446558634
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,fp8,4095,0.05745066702365875
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,8191,0.08935999870300293
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,fp8,8191,0.09943999846776326
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,1,0.012341332932313284
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,fp8,1,0.013408000270525614
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,3,0.012421333541472753
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,fp8,3,0.01340266689658165
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,7,0.012389333297808966
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,fp8,7,0.013573333621025085
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,15,0.012389333297808966
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,fp8,15,0.013455999394257864
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,31,0.012191999703645706
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,127,0.01320533330241839
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,fp8,31,0.013359999905029932
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,63,0.012746666868527731
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,fp8,63,0.013466666142145792
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,fp8,127,0.014991999914248785
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,255,0.01469333345691363
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,fp8,255,0.015594666202863058
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,511,0.016704000532627106
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,fp8,511,0.018170667191346485
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,1023,0.028586665789286297
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,fp8,1023,0.023269332945346832
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,2047,0.04414399961630503
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,fp8,2047,0.03675200045108795
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,4095,0.0745066652695338
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,fp8,4095,0.056645333766937256
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,8191,0.13552533586819968
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,1,0.01190399999419848
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,fp8,8191,0.10339200496673584
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,fp8,1,0.013034666577974955
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,3,0.01191466674208641
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,fp8,3,0.012954667210578918
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,7,0.01201066623131434
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,15,0.011941333611806234
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,fp8,7,0.013013333082199097
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,fp8,15,0.012960000584522883
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,31,0.011850666254758835
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,fp8,31,0.01303999995191892
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,63,0.011829332758982977
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,fp8,63,0.013253333667914072
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,127,0.011994666109482447
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,fp8,127,0.01443733274936676
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,255,0.01358933374285698
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,fp8,255,0.015050667027632395
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,511,0.015360000232855478
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,fp8,511,0.01749333366751671
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,1023,0.02004266654451688
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,fp8,1023,0.022495999932289124
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,1,0.01198400060335795
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,2047,0.02890666574239731
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,fp8,2047,0.03225066761175791
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,4095,0.04826133449872335
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,7,0.012229333321253458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,fp8,4095,0.05229333539803823
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,fp8,1,0.013050666699806849
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,3,0.012191999703645706
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,fp8,3,0.013088000317414602
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,fp8,7,0.013450667262077332
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,15,0.01179733375708262
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,fp8,15,0.01332266628742218
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,31,0.012453333785136541
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,63,0.012191999703645706
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,fp8,31,0.01369599997997284
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,fp8,63,0.013744000345468521
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,127,0.012261333564917246
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,fp8,127,0.01461333284775416
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,255,0.014202666779359182
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,fp8,255,0.015568000574906668
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,511,0.01603200038274129
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,fp8,511,0.017765333255132038
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,1023,0.0205226664741834
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,fp8,1023,0.02292266736427943
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,2047,0.033146666983763375
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,fp8,2047,0.03317866722742716
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,4095,0.052149335543314614
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,fp8,4095,0.05710933109124502
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,1,0.01259200026591619
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,fp8,1,0.013557333499193192
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,fp8,3,0.0136266661187013
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,3,0.012458667159080505
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,7,0.012448000411192576
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,fp8,31,0.013434667140245438
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,fp8,7,0.013733333597580591
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,15,0.01239466667175293
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,fp8,15,0.013546666751305262
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,31,0.01249066616098086
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,63,0.012645332763592402
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,fp8,63,0.013765333841244379
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,127,0.012800000607967377
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,fp8,127,0.014970666418472925
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,255,0.014485333114862442
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,fp8,255,0.015802666544914246
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,511,0.016229332735141117
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,fp8,511,0.018474667022625606
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,1023,0.028954667349656422
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,fp8,1023,0.023941333095232647
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,2047,0.044069334864616394
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,fp8,2047,0.03798400113979975
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,4095,0.07442133128643036
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,fp8,4095,0.05877333382765452
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,1,0.016480000068744022
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,fp8,1,0.017653333644072216
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,3,0.016623999923467636
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,fp8,3,0.017573333034912746
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,7,0.016607999801635742
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,fp8,7,0.01749333366751671
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,63,0.016789333273967106
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,15,0.016458666572968166
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,fp8,15,0.017605333278576534
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,31,0.01664000004529953
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,fp8,31,0.01764800027012825
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,fp8,63,0.01800000046690305
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,127,0.016879999389251072
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,fp8,127,0.020975999534130096
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,255,0.02019199977318446
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,fp8,255,0.021930667261282604
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,511,0.03152533372243246
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,fp8,511,0.027242665489514668
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,1023,0.04675200084845225
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,fp8,1023,0.041306667029857635
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,2047,0.07672533392906189
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,fp8,2047,0.06347733239332835
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,4095,0.13581867019335428
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,1,0.01128000020980835
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,fp8,4095,0.10477866729100545
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,1,0.011946666985750198
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,3,0.011055999745925268
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,3,0.012170666207869848
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,7,0.011264000087976456
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,7,0.0122079998254776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,15,0.01129066695769628
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,15,0.012240000069141388
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,31,0.011157333850860596
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,31,0.011994666109482447
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,63,0.011071999867757162
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,255,0.012357333054145178
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,63,0.012154666086037954
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,127,0.01108266661564509
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,127,0.013776000589132309
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,255,0.01350933313369751
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,511,0.012768000364303589
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,511,0.013584000368913015
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,1023,0.01302933320403099
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,1023,0.01440000037352244
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,2047,0.014149333039919535
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,2047,0.015365333606799444
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,4095,0.015696000307798386
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,4095,0.016688000410795212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,8191,0.017952000101407368
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,8191,0.01937066639463107
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,16383,0.022405333817005157
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,16383,0.024853333830833435
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,1,0.01102399950226148
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,1,0.012154666086037954
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,3,0.011157333850860596
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,3,0.012304000556468964
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,7,0.011168000598748526
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,7,0.012250666817029318
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,15,0.011034666250149408
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,15,0.012367999802033106
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,31,0.01110400011142095
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,31,0.011999999483426413
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,63,0.011322667201360067
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,63,0.01228800043463707
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,127,0.011045332998037338
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,127,0.013408000270525614
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,255,0.012517333030700684
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,255,0.013376000026861826
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,511,0.012448000411192576
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,511,0.013525333255529404
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,4095,0.016106666376193363
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,1023,0.01314666618903478
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,1023,0.014549333602190018
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,2047,0.014256000518798828
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,2047,0.01563199982047081
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,4095,0.017583999782800674
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,8191,0.019018666197856266
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,8191,0.020047999918460846
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,16383,0.022965334355831146
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,16383,0.025749333202838898
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,1,0.011359999577204386
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,1,0.012063999970753988
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,3,0.011215999722480774
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,3,0.012213333199421564
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,7,0.011343999455372492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,7,0.012341332932313284
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,15,0.011567999919255575
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,15,0.012458667159080505
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,31,0.011424000064531961
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,31,0.012122667084137598
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,63,0.011349332829316458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,63,0.012304000556468964
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,127,0.011402666568756104
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,127,0.013237333546082178
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,255,0.012469333906968435
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,255,0.013477332890033722
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,511,0.012773333738247553
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,511,0.014495999862750372
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,1023,0.013872000078360239
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,4095,0.01773333301146825
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,1023,0.015573333948850632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,2047,0.015493333339691162
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,2047,0.0163680004576842
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,4095,0.016480000068744022
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,8191,0.018687999496857326
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,8191,0.020186666399240494
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,16383,0.023647998770078022
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,16383,0.025792000194390614
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,1,0.011429333438475927
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,1,0.012256000190973282
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,3,0.01138666644692421
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,3,0.012421333541472753
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,7,0.011370666325092316
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,7,0.012175999581813812
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,15,0.01157333329319954
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,15,0.012245333443085352
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,31,0.011450666934251785
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,31,0.012335999558369318
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,63,0.011391999820868174
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,63,0.012240000069141388
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,127,0.011354666203260422
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,127,0.013546666751305262
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,255,0.012549333274364471
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,255,0.014197333405415217
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,511,0.013568000247081121
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,511,0.014511999984582266
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,1023,0.013663999736309052
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,1023,0.015024000157912573
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,2047,0.015722667177518208
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,2047,0.01670933390657107
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,4095,0.016575999557971954
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,4095,0.017935999979575474
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,8191,0.018895999838908512
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,8191,0.02067199970285098
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,16383,0.0306986669699351
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,1,0.011231999844312668
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,16383,0.02643200010061264
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,1,0.012042666474978128
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,3,0.01101333275437355
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,3,0.012223999947309494
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,7,0.01102399950226148
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,7,0.012266666938861212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,15,0.011285333583752314
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,15,0.012560000022252401
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,31,0.01110400011142095
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,31,0.012229333321253458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,63,0.011215999722480774
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,63,0.012437333663304647
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,127,0.011087999989589056
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,127,0.013679999858140945
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,255,0.012831999609867731
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,255,0.01341333364446958
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,511,0.012709333250919977
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,511,0.013936000565687815
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,1023,0.01381333296497663
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,1023,0.014853333433469137
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,2047,0.015557333827018738
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,8191,0.02351466566324234
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,2047,0.015978666643301647
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,4095,0.017610666652520496
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,4095,0.01870399961868922
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,8191,0.02086399992307027
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,16383,0.02978666623433431
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,16383,0.033301333586374916
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,1,0.011498666057984034
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,1,0.01221866657336553
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,3,0.011215999722480774
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,3,0.012223999947309494
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,7,0.011258666714032492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,7,0.012367999802033106
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,15,0.011178666104873022
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,15,0.012191999703645706
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,31,0.011434666812419891
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,31,0.012240000069141388
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,63,0.011237333218256632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,63,0.012346666306257248
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,127,0.011183999478816986
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,127,0.013242666920026144
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,255,0.01239466667175293
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,255,0.013477332890033722
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,511,0.013034666577974955
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,511,0.014661333213249842
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,1023,0.014282666146755219
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,1023,0.015872000406185787
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,2047,0.016085332880417507
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,2047,0.016602666427691776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,4095,0.018191999445358913
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,4095,0.01933866615096728
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,8191,0.021690666675567627
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,8191,0.023872000475724537
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,16383,0.031194667021433514
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,16383,0.03421866645415624
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,1,0.011343999455372492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,1,0.012202666451533636
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,3,0.011370666325092316
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,3,0.012261333564917246
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,7,0.011530666301647821
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,7,0.012175999581813812
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,15,0.011434666812419891
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,15,0.012346666306257248
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,31,0.011551999797423681
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,31,0.012202666451533636
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,63,0.011391999820868174
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,63,0.012319999436537424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,127,0.01166933278242747
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,127,0.013765333841244379
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,255,0.012794667234023413
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,255,0.014463999619086584
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,511,0.013253333667914072
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,511,0.014389333625634512
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,1023,0.01498666654030482
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,1023,0.01639466608564059
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,2047,0.016085332880417507
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,2047,0.017018667111794155
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,4095,0.01851733277241389
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,4095,0.019871999820073444
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,8191,0.02237333357334137
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,8191,0.024069334069887798
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,16383,0.032629333436489105
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,16383,0.03682666768630346
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,1,0.011877333124478659
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,1,0.012949333836634954
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,3,0.011648000528415045
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,3,0.012602667013804117
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,7,0.011776000261306763
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,7,0.01257066677014033
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,15,0.011600000162919363
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,15,0.012560000022252401
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,31,0.011717333147923151
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,31,0.012671999633312225
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,63,0.011727999895811081
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,63,0.012671999633312225
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,127,0.011823999385039011
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,127,0.014378666877746582
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,255,0.013077333569526672
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,255,0.01452800010641416
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,511,0.013525333255529404
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,511,0.015173333386580149
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,1023,0.015493333339691162
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,1023,0.016421332955360413
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,2047,0.017242666333913803
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,2047,0.016458666572968166
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,4095,0.019007999449968338
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,4095,0.020197333147128422
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,8191,0.030565333863099415
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,1,0.013125333935022354
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,8191,0.02465066562096278
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,16383,0.04610133171081543
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,16383,0.03974399964014689
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,fp8,1,0.013642666240533194
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,fp8,3,0.013829333086808523
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,3,0.013248000293970108
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,7,0.01293333371480306
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,fp8,7,0.013690666606028875
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,15,0.012837332983811697
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,fp8,15,0.013904000322024027
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,31,0.01312000056107839
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,fp8,31,0.01357866699496905
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,63,0.013178666432698568
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,fp8,63,0.013807999591032663
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,127,0.013487999637921652
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,fp8,127,0.01509333277742068
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,fp8,511,0.01838933303952217
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,255,0.014607999473810196
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,fp8,255,0.01595199977358182
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,511,0.016549333930015564
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,1023,0.021365332106749218
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,fp8,1023,0.023285334308942158
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,2047,0.032618666688601174
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,fp8,2047,0.03387733300526937
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,1,0.01310933381319046
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,fp8,1,0.014538666854302088
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,fp8,7,0.014650666465361914
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,3,0.013359999905029932
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,fp8,3,0.01469333345691363
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,7,0.013280000537633896
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,15,0.013210666676362356
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,fp8,15,0.014474666366974512
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,31,0.01322666679819425
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,fp8,31,0.014688000082969666
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,63,0.013327999661366144
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,fp8,63,0.014671999961137772
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,127,0.013770667215188345
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,fp8,127,0.015754666179418564
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,255,0.01524266724785169
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,fp8,255,0.016421332955360413
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,511,0.017477333545684814
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,fp8,511,0.019199999670187633
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,1023,0.02975466599067052
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,fp8,1023,0.024256000916163128
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,2047,0.04527466495831808
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,1,0.017125333348910015
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,fp8,2047,0.03881066789229711
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,fp8,1,0.018277333428462345
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,3,0.01681600014368693
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,fp8,3,0.01829333355029424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,fp8,15,0.018298666924238205
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,7,0.01720000058412552
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,31,0.017386666188637417
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,fp8,7,0.01836799954374631
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,15,0.017242666333913803
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,fp8,31,0.018186666071414948
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,63,0.01725333308180173
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,fp8,63,0.01836266616980235
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,127,0.017477333545684814
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,fp8,127,0.021066665649414062
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,255,0.020799999435742695
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,fp8,255,0.022511998812357586
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,511,0.031850665807724
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,fp8,511,0.028229333460330963
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,1023,0.04781866570313772
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,fp8,2047,0.06426133215427399
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,fp8,1023,0.04231466849644979
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,2047,0.07761600116888683
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,1,0.02404266595840454
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,fp8,1,0.026005332668622334
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,3,0.024271999796231587
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,fp8,3,0.025637333591779072
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,7,0.024127999941507976
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,fp8,15,0.025674665967623394
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,15,0.024458666642506916
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,fp8,31,0.02565866708755493
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,31,0.024357333779335022
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,fp8,7,0.025685332715511322
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,63,0.024373332659403484
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,fp8,63,0.025839999318122864
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,127,0.024832000335057575
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,fp8,127,0.030282666285832722
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,255,0.03682133307059606
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,fp8,255,0.03388266762097677
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,511,0.05147733290990194
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,fp8,511,0.050026665131251015
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,1023,0.08166933556397755
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,fp8,1023,0.06947733461856842
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,2047,0.13920000195503235
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,1,0.01109333336353302
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,fp8,2047,0.11235733826955159
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,3,0.011941333611806234
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,7,0.010591999938090643
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,1,0.01166933278242747
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,3,0.010687999427318573
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,7,0.01166933278242747
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,15,0.011173332730929056
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,15,0.011711999773979187
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,31,0.010826667149861654
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,31,0.011695999652147293
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,63,0.010778666784365972
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,63,0.011717333147923151
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,127,0.010869332899649939
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,127,0.013797332843144735
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,255,0.012319999436537424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,255,0.013461332768201828
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,511,0.014394666999578476
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,511,0.013317332913478216
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,1023,0.014890667051076889
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,1023,0.015850666910409927
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,2047,0.016117333124081295
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,2047,0.018687999496857326
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,4095,0.020138667275508244
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,4095,0.023002666731675465
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,8191,0.028933333853880566
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,8191,0.03285333265860876
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,16383,0.04677333434422811
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,16383,0.05202666421731313
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,1,0.011141333729028702
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,1,0.012479999413092932
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,3,0.011018666128317514
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,3,0.011898666620254517
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,7,0.010858666151762009
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,7,0.011877333124478659
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,15,0.011098666737476984
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,15,0.011893333246310553
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,31,0.011343999455372492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,31,0.01250133290886879
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,63,0.011071999867757162
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,63,0.012191999703645706
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,127,0.01101333275437355
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,511,0.015882667154073715
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,127,0.013162666310866674
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,255,0.012250666817029318
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,255,0.013946666071812311
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,511,0.013754667093356451
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,1023,0.01515199989080429
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,1023,0.0162773331006368
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,2047,0.01639466608564059
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,2047,0.01897066707412402
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,4095,0.02107733239730199
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,4095,0.023445333043734234
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,8191,0.030026666820049286
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,8191,0.03342933456103007
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,16383,0.0491893341143926
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,3,0.012144000579913458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,16383,0.05442666510740916
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,1,0.011349332829316458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,1,0.012351999680201212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,3,0.011477333803971609
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,7,0.011130666981140772
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,7,0.012389333297808966
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,15,0.011472000430027643
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,15,0.01210133358836174
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,31,0.011551999797423681
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,31,0.012154666086037954
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,63,0.011429333438475927
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,63,0.012319999436537424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,127,0.011429333438475927
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,127,0.013951999445756277
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,255,0.012997332960367203
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,255,0.014688000082969666
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,511,0.014671999961137772
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,511,0.015573333948850632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,1023,0.01562133307258288
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,1023,0.016538667182127636
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,2047,0.016938666502634685
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,2047,0.019274666905403137
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,4095,0.021151999632517498
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,4095,0.02372266600529353
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,8191,0.034117333590984344
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,8191,0.03634133438269297
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,16383,0.053317333261171974
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,16383,0.05813866853713989
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,1,0.011855999628702799
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,1,0.013199999928474426
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,3,0.011946666985750198
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,3,0.01303999995191892
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,7,0.01191466674208641
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,7,0.012543999900420507
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,15,0.011648000528415045
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,15,0.012778667112191519
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,31,0.011839999506870905
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,31,0.012837332983811697
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,63,0.011685332904259363
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,63,0.013274667163689932
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,127,0.011754666765530905
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,127,0.014170666535695394
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,255,0.013424000392357508
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,255,0.014752000570297241
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,511,0.015178666760524115
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,511,0.0161013330022494
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,1023,0.01621333385507266
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,1023,0.016762666404247284
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,2047,0.017093333105246227
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,4095,0.03068800022204717
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,2047,0.019440000255902607
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,4095,0.0244159996509552
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,8191,0.04550399879614512
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,8191,0.038848000268141426
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,16383,0.07644266883532207
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,16383,0.05973333120346069
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,1,0.014981333166360855
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,fp8,1,0.016362667083740234
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,3,0.015184000134468079
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,fp8,3,0.01653333380818367
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,7,0.01523200049996376
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,fp8,7,0.016613333175579708
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,15,0.015168000012636185
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,fp8,15,0.01640533283352852
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,31,0.015125333021084467
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,fp8,31,0.016501333564519882
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,63,0.015200000256299973
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,fp8,63,0.016517333686351776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,127,0.015754666179418564
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,fp8,127,0.017925333231687546
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,255,0.017717332889636356
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,fp8,255,0.01858666663368543
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,511,0.019968000551064808
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,fp8,511,0.02126399924357732
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,1023,0.031157332162062328
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,1,0.019088000059127808
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,fp8,1023,0.027727998793125153
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,fp8,1,0.020234666764736176
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,3,0.018810667097568512
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,fp8,3,0.020245333512624104
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,7,0.018661333868900936
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,fp8,7,0.020058666666348774
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,15,0.01897066707412402
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,fp8,15,0.020213333268960316
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,31,0.01887999971707662
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,fp8,31,0.020495999604463577
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,63,0.018757333358128864
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,fp8,127,0.02309333284695943
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,fp8,63,0.020319999506076176
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,255,0.02298133323589961
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,fp8,255,0.024613333245118458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,127,0.01903466631968816
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,511,0.0346666673819224
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,fp8,511,0.031130666534105938
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,1023,0.05009600023428599
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,3,0.025333332518736523
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,fp8,1023,0.045066664616266884
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,1,0.025653332471847534
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,fp8,1,0.027530667682488758
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,fp8,3,0.02722666660944621
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,31,0.025648000339667004
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,fp8,31,0.027802666028340656
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,fp8,63,0.027535999814669292
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,63,0.02571200082699458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,7,0.025546667476495106
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,fp8,7,0.027664000789324444
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,15,0.02587199956178665
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,fp8,15,0.027322667340437572
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,127,0.0266239990790685
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,fp8,511,0.052655999859174095
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,fp8,127,0.032602667808532715
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,255,0.039247999588648476
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,fp8,255,0.03601066768169403
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,fp8,1023,0.07186133166154225
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,511,0.053301334381103516
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,1023,0.0830506682395935
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,1,0.038373333712418876
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,3,0.03884266565243403
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,fp8,1,0.04162666698296865
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,fp8,3,0.04154666761557261
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,fp8,15,0.04166933397452036
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,fp8,7,0.04152533411979675
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,7,0.03828799972931544
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,31,0.039162665605545044
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,fp8,31,0.041690667470296226
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,15,0.03882133215665817
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,63,0.039503999054431915
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,fp8,63,0.04162666698296865
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,127,0.04580800235271454
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,fp8,127,0.05230399966239929
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,255,0.05922133227189382
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,fp8,255,0.06306666632493337
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,1023,0.14685333768526712
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,fp8,1023,0.12477866808573405
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,511,0.08987200260162354
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,fp8,511,0.0863200028737386
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,1,0.0225600004196167
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,fp8,1,0.02479466547568639
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,3,0.02253866692384084
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,7,0.022304000953833263
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,fp8,3,0.024725332856178284
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,fp8,15,0.02447466552257538
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,fp8,7,0.024527999262015026
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,15,0.022624000906944275
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,31,0.022991999983787537
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,fp8,31,0.024821333587169647
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,63,0.022650666534900665
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,fp8,63,0.02443733314673106
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,127,0.023039999107519787
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,fp8,127,0.02712533374627431
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,255,0.02757333219051361
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,fp8,255,0.02890666574239731
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,1,0.02897600084543228
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,fp8,1,0.03162133445342382
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,3,0.028736000259717304
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,fp8,3,0.03133333226044973
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,7,0.029029332101345062
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,fp8,7,0.03162666658560435
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,15,0.02914133419593175
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,fp8,15,0.03166933357715607
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,31,0.029333333174387615
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,fp8,31,0.03149333347876867
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,63,0.02898666759332021
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,fp8,63,0.03143999973932902
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,127,0.031311998764673867
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,fp8,127,0.03612799942493439
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,255,0.044256001710891724
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,1,0.042266666889190674
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,fp8,255,0.0403466671705246
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,fp8,1,0.045040001471837364
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,3,0.04187199970086416
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,fp8,3,0.04513599971930186
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,7,0.041850666205088295
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,fp8,7,0.04515199859937032
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,15,0.04238399863243103
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,fp8,15,0.04571199913819631
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,fp8,31,0.045279999574025474
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,31,0.042266666889190674
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,63,0.04460266729195913
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,fp8,63,0.04542933404445648
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,127,0.04974400003751119
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,fp8,127,0.05560000240802765
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,255,0.06527466575304668
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,1,0.06794666747252147
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,fp8,255,0.06751466790835063
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,fp8,1,0.07201066613197327
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,3,0.06771199901898702
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,fp8,3,0.07234666744867961
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,7,0.06788266698519389
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,fp8,7,0.07231466472148895
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,15,0.06818666557470958
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,fp8,15,0.07321600119272868
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,fp8,31,0.07227199772993724
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,31,0.06975999971230824
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,63,0.07434666653474171
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,fp8,63,0.07507733503977458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,127,0.0769706666469574
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,fp8,127,0.09412800272305806
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,255,0.10391466816266377
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,fp8,255,0.1099679966767629
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,1,0.04367466767628988
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,fp8,1,0.0450186679760615
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,3,0.04355733096599579
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,fp8,3,0.04516266783078512
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,7,0.0436160018046697
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,fp8,7,0.045237332582473755
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,15,0.04377600053946177
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,31,0.04386133452256521
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,fp8,15,0.045328001181284584
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,fp8,31,0.04539200166861216
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,63,0.04348800083001455
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,fp8,63,0.045168002446492515
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,127,0.04830400149027506
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,fp8,127,0.04925866425037384
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,1,0.05539200206597646
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,fp8,1,0.05789333085219065
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,3,0.05540800094604492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,fp8,3,0.05830933153629303
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,7,0.05518400172392527
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,fp8,7,0.057664001981417336
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,15,0.0558186670144399
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,fp8,15,0.057760000228881836
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,31,0.05625600119431814
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,fp8,31,0.05797333518664042
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,fp8,63,0.057616000374158226
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,63,0.06061866879463196
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,127,0.06572266419728597
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,fp8,127,0.06915199756622314
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,1,0.08180800080299377
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,fp8,1,0.08681066830952962
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,3,0.08216533561547597
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,fp8,3,0.08687466382980347
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,7,0.08226666847864787
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,fp8,7,0.08707732955614726
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,15,0.08278400202592213
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,fp8,15,0.08694932858149211
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,31,0.08773333827654521
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,fp8,31,0.08715200424194336
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,63,0.0902346670627594
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,fp8,63,0.0902239978313446
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,127,0.09277866284052531
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,1,0.13474133610725403
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,fp8,1,0.14004799723625183
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,fp8,127,0.1095413366953532
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,3,0.13338667154312134
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,fp8,3,0.14127999544143677
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,7,0.1339466671148936
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,fp8,7,0.1425920029481252
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,15,0.13935466607411703
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,fp8,15,0.14102933804194132
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,31,0.14166933298110962
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,1,0.011077333241701126
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,fp8,31,0.14433067043622336
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,63,0.14335999886194864
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,1,0.012341332932313284
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,3,0.012042666474978128
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,3,0.011109333485364914
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,7,0.010911999891201654
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,15,0.011237333218256632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,fp8,63,0.14787733554840088
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,15,0.012037333101034164
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,63,0.011359999577204386
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,7,0.012015999605258306
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,31,0.010965333630641302
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,127,0.011247999966144562
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,31,0.012138667205969492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,127,0.14831466476122537
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,63,0.012063999970753988
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,127,0.013418667018413544
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,fp8,127,0.1816693345705668
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,255,0.01221866657336553
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,255,0.014501333236694336
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,511,0.014064000298579534
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,511,0.015573333948850632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,1023,0.01821333294113477
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,2047,0.020538666596015293
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,1023,0.01602666700879733
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,4095,0.028991999725500744
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,4095,0.0328053335348765
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,8191,0.046944002310434975
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,2047,0.02313599983851115
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,8191,0.052282666166623436
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,16383,0.08416000008583069
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,16383,0.09532800316810608
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,1,0.011285333583752314
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,1,0.01232533281048139
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,3,0.011231999844312668
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,3,0.012175999581813812
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,7,0.011493333925803503
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,7,0.012293333808581034
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,15,0.012159999459981918
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,15,0.01246400053302447
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,31,0.011333333949247995
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,31,0.012298667182525
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,63,0.011237333218256632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,63,0.012213333199421564
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,127,0.011440000186363855
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,127,0.013445333888133367
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,1023,0.01634666696190834
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,255,0.01268799975514412
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,255,0.01584533353646596
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,511,0.01431999976436297
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,511,0.016389333953460056
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,1023,0.018543999642133713
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,8191,0.04959466556708018
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,2047,0.020799999435742695
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,2047,0.023455999791622162
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,4095,0.029338667790095013
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,4095,0.03338133295377096
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,8191,0.054010664423306785
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,16383,0.08405866225560506
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,1,0.012063999970753988
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,16383,0.10068800052007039
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,1,0.01339200014869372
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,3,0.011695999652147293
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,3,0.012719999998807907
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,7,0.011765333513418833
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,7,0.012768000364303589
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,15,0.011648000528415045
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,15,0.012954667210578918
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,31,0.011781333635250727
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,31,0.012869333227475485
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,63,0.011770666887362799
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,63,0.012975999464591345
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,127,0.01179733375708262
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,127,0.014736000448465347
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,255,0.013418667018413544
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,255,0.015504000087579092
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,511,0.016037333756685257
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,1023,0.01674666628241539
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,511,0.016442666451136272
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,1023,0.01841066653529803
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,2047,0.02120000123977661
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,2047,0.023797333240509033
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,4095,0.03289599965016047
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,8191,0.05798399945100149
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,4095,0.03363200028737386
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,8191,0.05494933327039083
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,16383,0.11114666859308879
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,16383,0.10035199920336406
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,1,0.012074666718641916
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,1,0.013418667018413544
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,3,0.011727999895811081
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,3,0.0129120002190272
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,7,0.01179733375708262
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,7,0.013061333447694778
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,15,0.0120319997270902
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,15,0.01302933320403099
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,31,0.012181332955757776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,31,0.01314666618903478
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,63,0.011658667276302973
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,63,0.013199999928474426
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,127,0.012282667060693106
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,127,0.014533333480358124
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,255,0.013365333278973898
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,255,0.016501333564519882
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,511,0.015200000256299973
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,511,0.01775466650724411
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,1023,0.0173333336909612
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,1023,0.01921066641807556
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,2047,0.03029866764942805
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,2047,0.025568000972270966
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,4095,0.04545066754023234
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,4095,0.03938666731119156
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,8191,0.07649066547552745
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,8191,0.060415998101234436
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,16383,0.13642133275667825
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,16383,0.10192533334096272
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,fp8,1,0.039477333426475525
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,1,0.03870933254559835
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,3,0.03882133215665817
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,fp8,3,0.04009599983692169
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,fp8,7,0.039333333571751915
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,7,0.038719999293486275
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,15,0.0405173326532046
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,fp8,15,0.03966933240493139
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,31,0.042677332957585655
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,fp8,31,0.03977599988381068
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,1,0.06514666477839152
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,63,0.046800002455711365
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,fp8,1,0.06329600016276042
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,fp8,63,0.04074666649103165
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,3,0.06550399959087372
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,fp8,3,0.06365866462389629
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,7,0.06597333153088887
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,fp8,7,0.06306133170922597
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,15,0.0697813332080841
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,fp8,15,0.06402666866779327
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,31,0.07326933244864146
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,fp8,31,0.06589866677920024
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,63,0.07407466570536296
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,fp8,63,0.07262933254241943
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,1,0.116047998269399
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,fp8,1,0.113946666320165
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,3,0.1181066632270813
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,fp8,3,0.11425600449244182
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,7,0.12317867080370586
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,15,0.12899200121561685
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,fp8,7,0.11548266808191936
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,fp8,15,0.1172160009543101
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,31,0.12914666533470154
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,fp8,31,0.1255573332309723
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,63,0.13106667002042136
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,1,0.21333332856496176
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,fp8,1,0.20715200901031494
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,fp8,63,0.12784533699353537
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,fp8,3,0.20835200945536295
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,3,0.22519999742507935
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,7,0.23084799448649088
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,fp8,7,0.21056000391642252
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,15,0.23220266898473105
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,fp8,15,0.22386133670806885
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,31,0.23358933130900064
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,fp8,31,0.2239733338356018
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,63,0.23617599407831827
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,fp8,63,0.22984000047047934
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,float16,1,0.07713599999745686
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,fp8,1,0.06990933418273926
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,float16,3,0.07693333427111308
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,fp8,3,0.06984533369541168
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,float16,7,0.07628266513347626
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,fp8,7,0.07091733316580455
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,float16,15,0.0763679991165797
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,fp8,15,0.07643733421961467
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,fp8,31,0.07780799766381581
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,float16,1,0.1311199963092804
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,float16,31,0.07713599999745686
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,fp8,1,0.11945600310961406
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,fp8,3,0.12180800239245097
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,float16,3,0.13152000308036804
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,float16,7,0.13302399714787802
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,fp8,7,0.1237600048383077
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,float16,15,0.13276267051696777
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,fp8,15,0.12660800417264303
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,fp8,31,0.12807466586430868
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,float16,31,0.13212800025939941
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,float16,1,0.24198933442433676
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,fp8,1,0.2260319987932841
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,float16,3,0.24445867538452148
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,fp8,3,0.23061333100001016
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,float16,7,0.24277333418528238
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,fp8,7,0.23207465807596842
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,float16,15,0.24409067630767822
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,float16,31,0.24433066447575888
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,fp8,15,0.23372799158096313
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,fp8,31,0.23613333702087402
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,float16,1,0.45051733652750653
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,fp8,1,0.42534399032592773
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,float16,3,0.44867201646169025
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,fp8,3,0.42419199148813885
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,fp8,7,0.42820799350738525
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,1,0.011514666179815928
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,fp8,1,0.012223999947309494
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,3,0.011477333803971609
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,fp8,3,0.012319999436537424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,7,0.011530666301647821
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,fp8,7,0.012335999558369318
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,float16,15,0.4482826789220174
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,15,0.011445333560307821
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,fp8,15,0.012442667037248611
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,fp8,15,0.43011200428009033
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,31,0.011557333171367645
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,fp8,31,0.012213333199421564
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,fp8,63,0.012351999680201212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,63,0.011621333658695221
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,fp8,127,0.013568000247081121
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,255,0.012762666990359625
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,127,0.01184533288081487
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,fp8,255,0.014416000495354334
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,511,0.014954666296641031
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,fp8,511,0.01706133286158244
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,1023,0.01932799940307935
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,fp8,1023,0.02184533327817917
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,2047,0.02757866680622101
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,fp8,2047,0.03152533372243246
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,4095,0.04621333380540212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,fp8,4095,0.05100266635417938
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,8191,0.08318933347860973
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,fp8,8191,0.09242666761080424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,1,0.011855999628702799
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,fp8,1,0.012928000340859095
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,3,0.011594666788975397
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,float16,31,0.45480533440907794
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,fp8,3,0.012853333105643591
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,7,0.011727999895811081
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,fp8,7,0.013408000270525614
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,15,0.011909333368142446
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,fp8,31,0.43275733788808185
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,fp8,15,0.012874666601419449
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,31,0.01163200040658315
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,63,0.011621333658695221
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,fp8,63,0.013343999783198038
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,fp8,31,0.013125333935022354
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,float16,7,0.4503946701685588
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,127,0.011749333391586939
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,fp8,127,0.01421333352724711
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,fp8,255,0.01552533358335495
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,511,0.01533866673707962
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,255,0.013370666652917862
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,fp8,2047,0.03239466746648153
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,fp8,511,0.017658667018016178
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,1023,0.01995733380317688
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,fp8,1023,0.022517333428064983
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,2047,0.028832000990708668
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,4095,0.048656001687049866
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,fp8,4095,0.05449600021044413
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,8191,0.08273600041866302
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,fp8,8191,0.09898133079210918
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,1,0.012074666718641916
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,fp8,1,0.013280000537633896
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,3,0.01211200033624967
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,fp8,3,0.013514666507641474
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,7,0.012272000312805176
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,fp8,7,0.013130666067202887
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,15,0.012080000092585882
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,fp8,15,0.013248000293970108
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,31,0.012159999459981918
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,fp8,31,0.013568000247081121
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,63,0.011941333611806234
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,fp8,63,0.013232000172138214
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,127,0.012351999680201212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,fp8,127,0.014416000495354334
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,255,0.013487999637921652
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,1023,0.019973333925008774
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,fp8,255,0.015247999380032221
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,511,0.015722667177518208
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,fp8,511,0.017877332866191864
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,fp8,1023,0.022522665560245514
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,2047,0.032672000428040825
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,fp8,2047,0.03283733377854029
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,4095,0.05305600166320801
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,fp8,4095,0.0568800022204717
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,8191,0.09264533718427022
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,3,0.012144000579913458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,fp8,8191,0.09918933113416036
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,1,0.012261333564917246
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,fp8,1,0.013557333499193192
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,fp8,3,0.013482666263977686
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,7,0.012186666329701742
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,fp8,7,0.013493333011865616
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,15,0.012165332833925882
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,fp8,15,0.013482666263977686
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,31,0.012458667159080505
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,fp8,31,0.013482666263977686
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,63,0.012527999778588613
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,fp8,63,0.01370666672786077
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,511,0.01579733317097028
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,127,0.012698666503032049
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,fp8,127,0.015290666371583939
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,255,0.014069333672523499
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,fp8,255,0.015664000064134598
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,fp8,511,0.01830400029818217
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,1023,0.028746667007605236
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,fp8,1023,0.023765332996845245
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,2047,0.04372266431649526
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,fp8,2047,0.038149334490299225
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,4095,0.07506666580835979
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,fp8,4095,0.05876266459623972
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,8191,0.13352533181508383
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,fp8,8191,0.10065600275993347
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,1,0.012175999581813812
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,fp8,1,0.012949333836634954
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,3,0.012058666596810022
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,fp8,3,0.012815999488035837
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,7,0.012005332857370377
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,fp8,7,0.012821332861979803
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,15,0.011829332758982977
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,fp8,15,0.012800000607967377
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,31,0.012165332833925882
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,fp8,31,0.012778667112191519
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,63,0.012080000092585882
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,fp8,63,0.01267733300725619
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,127,0.012058666596810022
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,fp8,127,0.014746667196353277
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,255,0.013434667140245438
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,fp8,255,0.015487999965747198
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,511,0.015504000087579092
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,fp8,511,0.017674667139848072
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,1023,0.019653332730134327
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,fp8,1023,0.022229333718617756
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,2047,0.029205332199732464
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,fp8,2047,0.032245332996050514
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,4095,0.04822933177153269
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,fp8,4095,0.05266133447488149
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,1,0.012047999848922094
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,fp8,1,0.012794667234023413
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,3,0.011973333855470022
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,fp8,3,0.013455999394257864
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,7,0.01232533281048139
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,fp8,7,0.012842666357755661
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,15,0.011855999628702799
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,fp8,15,0.013370666652917862
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,31,0.01198400060335795
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,fp8,31,0.01293333371480306
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,63,0.011994666109482447
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,fp8,63,0.013584000368913015
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,127,0.01251199965675672
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,fp8,127,0.014458666245142618
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,255,0.014069333672523499
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,fp8,255,0.015568000574906668
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,511,0.01581866666674614
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,fp8,511,0.018426666657129925
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,1023,0.020288000504175823
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,fp8,1023,0.02275199939807256
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,2047,0.03317866722742716
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,fp8,2047,0.033173332611719765
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,4095,0.06162666777769724
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,fp8,4095,0.056986664732297264
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,1,0.012373333175977072
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,fp8,1,0.013546666751305262
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,fp8,7,0.013424000392357508
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,3,0.012309333930412928
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,fp8,3,0.013568000247081121
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,7,0.012506666282812754
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,15,0.012383999923865
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,fp8,15,0.013445333888133367
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,31,0.012549333274364471
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,fp8,31,0.013679999858140945
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,63,0.012416000167528788
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,fp8,63,0.013818666338920593
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,127,0.012778667112191519
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,fp8,127,0.015135999768972397
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,255,0.014218666901191076
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,fp8,255,0.015919999529918034
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,511,0.016447999825080235
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,fp8,511,0.018511999398469925
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,1023,0.029829333225886028
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,fp8,1023,0.023941333095232647
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,2047,0.04422933359940847
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,fp8,2047,0.03749866783618927
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,4095,0.07454933226108551
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,fp8,1,0.018559999763965607
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,fp8,3,0.017573333034912746
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,fp8,4095,0.058837334314982094
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,1,0.016442666451136272
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,3,0.016480000068744022
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,7,0.01642666632930438
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,fp8,7,0.017808000246683758
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,15,0.016575999557971954
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,fp8,15,0.017978666971127193
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,31,0.016528000434239704
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,fp8,31,0.017770666629076004
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,fp8,127,0.02067733307679494
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,63,0.016672000288963318
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,255,0.020346666375796
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,fp8,63,0.017893332988023758
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,127,0.016949333250522614
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,fp8,255,0.022128000855445862
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,511,0.030928000807762146
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,fp8,511,0.02804800122976303
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,1023,0.046522667010625206
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,fp8,1023,0.04171733558177948
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,fp8,2047,0.06307733555634816
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,2047,0.07749866445859273
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,3,0.010970667004585266
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,4095,0.13500799735387167
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,fp8,4095,0.10517866412798564
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,7,0.01201066623131434
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,1,0.011114666859308878
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,1,0.012149333953857422
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,31,0.011264000087976456
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,3,0.012159999459981918
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,7,0.011285333583752314
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,15,0.011215999722480774
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,15,0.012117333710193634
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,31,0.01232533281048139
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,63,0.011242666592200598
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,63,0.01210133358836174
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,127,0.01129066695769628
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,127,0.013525333255529404
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,255,0.012389333297808966
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,255,0.013663999736309052
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,511,0.012671999633312225
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,511,0.013754667093356451
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,1023,0.01293333371480306
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,1023,0.014117332796255747
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,2047,0.014298666268587112
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,2047,0.015402667224407196
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,4095,0.015418666104475657
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,4095,0.016714667280515034
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,8191,0.01770666614174843
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,8191,0.019189332922299702
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,16383,0.022266666094462078
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,16383,0.02465066562096278
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,1,0.011440000186363855
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,1,0.012240000069141388
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,3,0.011296000331640244
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,3,0.012432000289360682
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,7,0.011087999989589056
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,7,0.012047999848922094
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,15,0.011146667102972666
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,15,0.012538666526476542
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,31,0.011418666690587997
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,31,0.012202666451533636
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,63,0.011359999577204386
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,63,0.012453333785136541
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,127,0.011136000355084738
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,127,0.013194666554530462
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,255,0.012154666086037954
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,255,0.013376000026861826
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,511,0.012703999876976013
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,511,0.01351999988158544
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,4095,0.01565333331624667
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,1023,0.01292266696691513
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,1023,0.014762666076421738
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,2047,0.01440000037352244
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,2047,0.01570133368174235
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,4095,0.01758933315674464
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,8191,0.018474667022625606
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,8191,0.019546666493018467
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,16383,0.02293333411216736
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,16383,0.025274666647116344
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,1,0.011589333415031433
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,1,0.012170666207869848
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,3,0.011141333729028702
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,3,0.012272000312805176
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,7,0.011370666325092316
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,7,0.012202666451533636
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,15,0.011328000575304031
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,15,0.01257066677014033
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,31,0.011402666568756104
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,31,0.01227733368674914
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,63,0.011231999844312668
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,63,0.012362666428089142
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,127,0.01126933346192042
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,127,0.013248000293970108
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,255,0.012576000144084295
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,255,0.013514666507641474
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,511,0.012858666479587555
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,511,0.014522666732470194
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,1023,0.013845333208640417
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,1023,0.015466666469971338
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,2047,0.014970666418472925
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,2047,0.01693333312869072
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,4095,0.016735999534527462
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,4095,0.017237332959969837
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,8191,0.01852799952030182
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,8191,0.020367999871571858
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,16383,0.024080000817775726
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,16383,0.025642665723959606
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,1,0.011450666934251785
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,1,0.012202666451533636
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,3,0.011301333705584208
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,3,0.01228800043463707
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,7,0.011509332805871964
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,7,0.01232533281048139
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,15,0.011365332951148352
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,15,0.01259200026591619
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,31,0.011434666812419891
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,31,0.012298667182525
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,63,0.011525332927703857
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,63,0.012351999680201212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,127,0.011407999942700068
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,127,0.01332266628742218
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,255,0.012906666845083237
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,255,0.014549333602190018
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,511,0.013455999394257864
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,511,0.015024000157912573
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,1023,0.013850666582584381
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,1023,0.01543466622630755
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,2047,0.015317333241303762
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,2047,0.01637866720557213
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,4095,0.016714667280515034
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,16383,0.02588266630967458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,4095,0.01781333362062772
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,8191,0.01869333287080129
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,8191,0.020309332758188248
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,16383,0.030906667311986286
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,1,0.011071999867757162
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,1,0.01221866657336553
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,3,0.011050666371981302
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,3,0.012298667182525
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,7,0.01109333336353302
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,7,0.01228800043463707
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,15,0.01118933285276095
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,31,0.01129066695769628
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,15,0.012367999802033106
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,31,0.012159999459981918
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,63,0.011066666493813196
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,63,0.012245333443085352
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,127,0.011050666371981302
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,127,0.013455999394257864
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,255,0.012309333930412928
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,255,0.013541333377361298
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,511,0.012517333030700684
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,511,0.013829333086808523
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,1023,0.013999999811251959
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,1023,0.01479999969402949
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,2047,0.015658666690190632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,2047,0.016186666985352833
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,4095,0.01801066721479098
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,4095,0.01869333287080129
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,8191,0.021151999632517498
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,8191,0.023050665855407715
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,16383,0.029504001140594482
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,16383,0.03339733431736628
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,1,0.011231999844312668
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,1,0.012448000411192576
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,15,0.011429333438475927
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,3,0.011247999966144562
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,3,0.012256000190973282
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,7,0.011493333925803503
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,7,0.012202666451533636
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,15,0.012319999436537424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,31,0.011114666859308878
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,31,0.01249066616098086
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,63,0.01137599969903628
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,63,0.012181332955757776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,127,0.011407999942700068
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,127,0.013514666507641474
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,255,0.012421333541472753
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,255,0.01369599997997284
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,511,0.012954667210578918
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,511,0.014607999473810196
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,4095,0.01788266624013583
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,1023,0.014261333892742792
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,1023,0.01611199975013733
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,4095,0.01923199991385142
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,2047,0.016010666886965435
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,2047,0.016794666647911072
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,8191,0.021290667355060577
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,8191,0.023770667612552643
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,16383,0.03027733415365219
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,16383,0.03397866586844126
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,1,0.01146666705608368
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,1,0.012373333175977072
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,3,0.011488000551859537
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,3,0.01231466606259346
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,7,0.01128000020980835
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,7,0.012655999511480331
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,15,0.01156266654531161
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,15,0.012256000190973282
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,31,0.011301333705584208
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,31,0.012560000022252401
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,63,0.011258666714032492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,63,0.012202666451533636
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,127,0.011658667276302973
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,127,0.013781332721312841
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,255,0.012821332861979803
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,255,0.01431999976436297
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,511,0.013621332744757334
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,511,0.014848000059525171
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,1023,0.015050667027632395
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,1023,0.015935999651749928
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,2047,0.016538667182127636
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,2047,0.017029333859682083
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,4095,0.0185759998857975
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,4095,0.019871999820073444
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,8191,0.02260799954334895
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,8191,0.024293333292007446
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,16383,0.03329599897066752
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,16383,0.0344106654326121
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,1,0.01179733375708262
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,1,0.012693333129088083
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,3,0.011525332927703857
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,3,0.012703999876976013
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,7,0.011781333635250727
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,7,0.012853333105643591
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,15,0.011477333803971609
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,15,0.012351999680201212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,31,0.011584000041087469
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,31,0.012778667112191519
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,63,0.011882666498422623
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,63,0.012757333616415659
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,127,0.01198400060335795
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,127,0.014197333405415217
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,255,0.013199999928474426
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,255,0.0145066666106383
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,511,0.013893333574136099
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,511,0.015098666151364645
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,1023,0.015487999965747198
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,1023,0.016399999459584553
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,2047,0.016688000410795212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,2047,0.017221332838137943
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,4095,0.018874666343132656
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,4095,0.020096000283956528
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,8191,0.0305173322558403
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,8191,0.02478933334350586
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,16383,0.04612799982229868
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,16383,0.03914133210976919
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,1,0.012954667210578918
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,fp8,1,0.013690666606028875
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,3,0.012634667257467905
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,fp8,3,0.013573333621025085
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,7,0.013013333082199097
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,fp8,7,0.01351999988158544
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,15,0.013002666334311167
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,fp8,15,0.013536000003417334
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,31,0.012826666235923767
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,fp8,31,0.013861333330472311
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,63,0.012960000584522883
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,fp8,63,0.01370666672786077
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,127,0.013349333157142004
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,fp8,127,0.014954666296641031
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,255,0.014746667196353277
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,fp8,255,0.015882667154073715
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,511,0.016623999923467636
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,fp8,511,0.018437333405017853
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,1023,0.02096533278624217
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,fp8,1023,0.022858666876951855
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,2047,0.03364799916744232
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,fp8,2047,0.034101332227389015
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,1,0.013141332815090815
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,fp8,1,0.014314666390419006
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,3,0.01332266628742218
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,fp8,3,0.014416000495354334
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,fp8,15,0.014522666732470194
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,7,0.013007999708255133
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,fp8,7,0.0143306665122509
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,15,0.01332266628742218
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,31,0.01328533391157786
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,fp8,31,0.014042666802803675
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,63,0.013343999783198038
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,fp8,63,0.014335999886194864
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,127,0.013477332890033722
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,fp8,127,0.015583999454975128
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,255,0.014991999914248785
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,fp8,255,0.016517333686351776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,511,0.017498667041460674
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,fp8,511,0.019050666441520054
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,1023,0.02975466599067052
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,fp8,1023,0.02497600018978119
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,2047,0.04462933540344238
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,1,0.017114666601022083
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,fp8,2047,0.0388373335202535
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,fp8,1,0.0182239996890227
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,3,0.01695466662446658
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,fp8,3,0.01820266619324684
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,7,0.017152000218629837
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,15,0.017114666601022083
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,fp8,15,0.018101333330074947
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,fp8,7,0.01814933369557063
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,31,0.01714133347074191
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,fp8,31,0.018266666680574417
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,63,0.01717866708834966
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,fp8,63,0.018437333405017853
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,127,0.01757866640885671
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,fp8,127,0.021055998901526134
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,255,0.02075200031201045
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,fp8,255,0.022309333086013794
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,511,0.03253333270549774
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,1023,0.048170665899912514
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,fp8,511,0.02789866675933202
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,fp8,1023,0.04257600009441376
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,2047,0.07706133524576823
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,fp8,2047,0.06419733166694641
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,1,0.024143998821576435
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,fp8,1,0.025674665967623394
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,3,0.02420266717672348
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,fp8,3,0.025311999022960663
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,fp8,7,0.0252960001428922
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,7,0.024282666544119518
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,15,0.024400000770886738
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,fp8,15,0.025546667476495106
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,31,0.024277334411938984
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,fp8,31,0.025781333446502686
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,63,0.024186665813128155
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,fp8,63,0.02535466601451238
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,127,0.024826665719350178
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,fp8,127,0.03014400104681651
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,fp8,255,0.034234667817751564
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,255,0.037045332292715706
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,511,0.05129066606362661
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,1023,0.0813973347345988
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,fp8,511,0.04969066878159841
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,fp8,1023,0.06951466699441274
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,2047,0.1381226678689321
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,1,0.0106133334338665
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,fp8,2047,0.11205333471298218
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,1,0.0116799995303154
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,3,0.010741333166758219
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,3,0.011509332805871964
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,7,0.010858666151762009
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,7,0.011418666690587997
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,15,0.011642667154471079
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,15,0.01110400011142095
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,31,0.010751999914646149
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,31,0.011637333780527115
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,63,0.010816000401973724
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,63,0.011567999919255575
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,127,0.010778666784365972
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,127,0.013104000439246496
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,255,0.01211200033624967
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,255,0.013610667238632837
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,511,0.013418667018413544
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,511,0.014597332725922266
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,1023,0.014778666198253632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,1023,0.015344000111023584
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,2047,0.015765332927306492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,2047,0.018112000077962875
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,4095,0.020245333512624104
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,4095,0.023082666099071503
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,8191,0.02884799987077713
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,8191,0.03283733377854029
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,16383,0.04741866886615753
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,16383,0.05219733218352
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,1,0.011007999380429586
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,1,0.011839999506870905
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,3,0.010869332899649939
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,3,0.012437333663304647
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,7,0.011007999380429586
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,7,0.011727999895811081
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,15,0.01091733326514562
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,15,0.011839999506870905
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,31,0.01101333275437355
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,31,0.011584000041087469
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,63,0.010911999891201654
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,63,0.012416000167528788
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,127,0.010992000500361124
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,127,0.01332266628742218
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,255,0.012805332740147909
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,255,0.014080000420411428
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,511,0.013722666849692663
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,511,0.015344000111023584
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,1023,0.015546667079130808
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,1023,0.01676799977819125
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,2047,0.01637866720557213
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,2047,0.01874133323629697
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,4095,0.02130666623512904
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,4095,0.023621333142121632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,8191,0.030031998952229817
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,8191,0.03374933451414108
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,1,0.012309333930412928
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,16383,0.04930666585763296
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,16383,0.05295999844868978
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,1,0.011349332829316458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,3,0.011183999478816986
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,3,0.01201066623131434
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,7,0.011359999577204386
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,7,0.01240533341964086
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,15,0.011429333438475927
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,15,0.01209066684047381
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,31,0.011424000064531961
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,31,0.012181332955757776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,63,0.011237333218256632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,63,0.01227733368674914
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,127,0.011530666301647821
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,127,0.014618666221698126
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,255,0.013295999417702356
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,255,0.014549333602190018
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,511,0.014677333335081736
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,511,0.01603200038274129
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,1023,0.015919999529918034
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,1023,0.01674666628241539
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,2047,0.017029333859682083
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,2047,0.01926933353145917
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,4095,0.021551998953024547
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,4095,0.023818666736284893
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,8191,0.03338133295377096
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,8191,0.03645866612593333
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,16383,0.06260799864927928
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,16383,0.05851200222969055
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,1,0.011503999431928
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,1,0.012757333616415659
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,3,0.011685332904259363
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,3,0.012730666746695837
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,7,0.011578666667143503
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,7,0.012837332983811697
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,15,0.011946666985750198
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,15,0.013461332768201828
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,31,0.011898666620254517
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,31,0.012618667135636011
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,63,0.011642667154471079
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,63,0.012858666479587555
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,127,0.011722666521867117
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,127,0.014522666732470194
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,255,0.013210666676362356
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,255,0.01482133318980535
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,511,0.014373333503802618
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,511,0.016224000602960587
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,1023,0.016143999993801117
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,1023,0.016778666526079178
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,2047,0.01720000058412552
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,2047,0.01932266727089882
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,4095,0.03046400099992752
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,4095,0.026149332523345947
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,8191,0.04587733248869578
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,8191,0.038746667404969536
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,16383,0.07554133236408234
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,16383,0.05982933441797892
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,1,0.015013333410024643
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,fp8,1,0.0161920003592968
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,3,0.014991999914248785
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,fp8,3,0.015989333391189575
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,7,0.015226667126019796
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,fp8,7,0.015989333391189575
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,15,0.014906667172908783
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,fp8,15,0.016095999628305435
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,31,0.014975999792416891
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,fp8,31,0.016202667107184727
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,63,0.01504533365368843
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,fp8,63,0.015856000284353893
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,127,0.015311999867359797
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,fp8,127,0.017375999440749485
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,255,0.017450666675964992
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,fp8,255,0.018266666680574417
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,511,0.019226666539907455
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,fp8,511,0.021162666380405426
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,1023,0.03155199935038885
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,1,0.018464000274737675
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,fp8,1023,0.026335999369621277
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,fp8,3,0.019946667055288952
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,fp8,1,0.01961600035429001
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,3,0.01846933364868164
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,7,0.018272000054518383
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,fp8,7,0.019946667055288952
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,15,0.01848000039656957
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,31,0.018543999642133713
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,fp8,15,0.01982933282852173
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,fp8,31,0.020106667031844456
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,63,0.018506667266289394
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,fp8,127,0.022954667607943218
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,fp8,63,0.019925333559513092
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,127,0.018789333601792652
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,255,0.022778667509555817
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,fp8,255,0.024383999407291412
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,511,0.0346666673819224
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,fp8,511,0.030576000610987347
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,1023,0.049839998284975685
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,fp8,1023,0.04457599918047587
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,1,0.025205334027608235
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,fp8,1,0.02719466636578242
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,3,0.025237334271272022
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,fp8,3,0.027130665878454845
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,7,0.025226667523384094
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,fp8,15,0.02700799951950709
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,15,0.025349333882331848
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,fp8,7,0.02719466636578242
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,31,0.025461333493391674
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,fp8,31,0.027679999669392902
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,63,0.025397333006064098
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,fp8,63,0.027322667340437572
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,127,0.026543999711672466
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,fp8,127,0.031845333675543465
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,255,0.039103999733924866
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,fp8,255,0.03604800005753835
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,511,0.05243733525276184
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,fp8,511,0.0516480008761088
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,1,0.03801066676775614
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,fp8,1,0.04117333392302195
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,3,0.03824000060558319
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,1023,0.08308800061543782
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,fp8,1023,0.07121066749095917
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,fp8,3,0.04196799794832865
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,fp8,7,0.04091733445723852
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,7,0.03823466598987579
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,15,0.038592000802357994
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,fp8,15,0.04144000013669332
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,63,0.03946666667858759
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,fp8,63,0.041120000183582306
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,fp8,127,0.05124799907207489
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,127,0.044863998889923096
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,fp8,31,0.04110399881998698
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,31,0.03862933317820231
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,fp8,255,0.06284266710281372
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,1023,0.1458399991194407
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,255,0.061237335205078125
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,fp8,1023,0.12433066964149475
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,1,0.022282667458057404
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,fp8,1,0.02387733260790507
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,fp8,7,0.023962666591008503
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,15,0.021920000513394673
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,fp8,3,0.023733332753181458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,3,0.02233600119749705
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,fp8,511,0.08427199721336365
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,fp8,15,0.0239680012067159
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,7,0.022389332453409832
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,fp8,31,0.02365333338578542
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,31,0.022250667214393616
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,511,0.08941866954167683
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,63,0.022298666338125866
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,fp8,63,0.023941333095232647
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,127,0.022986667851607006
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,fp8,127,0.02649066597223282
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,fp8,255,0.028149334092934925
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,255,0.02701866626739502
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,1,0.02861333390076955
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,3,0.02870933214823405
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,fp8,1,0.030613332986831665
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,fp8,3,0.03073066721359889
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,7,0.02961066613594691
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,fp8,7,0.030591999491055805
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,15,0.028607999285062153
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,fp8,15,0.030597334106763203
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,31,0.028789333999156952
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,fp8,31,0.030613332986831665
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,63,0.029071999092896778
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,fp8,63,0.031082667410373688
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,127,0.029989334444204967
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,fp8,127,0.03513066718975703
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,255,0.04298133154710134
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,1,0.041759997606277466
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,fp8,255,0.04127466678619385
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,fp8,1,0.0447573314110438
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,3,0.04179200033346812
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,fp8,3,0.04543466866016388
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,7,0.041509332756201424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,fp8,7,0.04474666714668274
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,15,0.041722665230433144
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,fp8,15,0.04491733511288961
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,31,0.042170668641726174
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,fp8,31,0.04483200112978617
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,63,0.043290664752324425
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,fp8,63,0.045423999428749084
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,127,0.04940799872080485
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,fp8,127,0.05568000177542368
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,255,0.06270933151245117
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,fp8,255,0.06657066444555919
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,1,0.06758399804433186
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,fp8,1,0.07188799977302551
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,3,0.06770133475462596
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,fp8,3,0.07253333429495494
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,7,0.06774400174617767
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,fp8,7,0.07191999753316243
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,15,0.06782933572928111
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,fp8,31,0.07210666437943776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,31,0.06956266860167186
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,fp8,15,0.07187200089295705
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,63,0.07438399891058604
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,fp8,63,0.07357866565386455
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,127,0.07765333354473114
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,fp8,127,0.09364799658457439
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,255,0.1030560036500295
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,fp8,255,0.11020267009735107
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,1,0.040991999208927155
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,fp8,1,0.04228266576925913
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,3,0.041002665956815086
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,fp8,3,0.042597333590189614
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,7,0.040896000961462654
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,fp8,7,0.04244266450405121
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,fp8,15,0.04260799785455068
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,31,0.0410453329483668
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,15,0.040965333580970764
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,fp8,31,0.04241600135962168
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,63,0.041365332901477814
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,fp8,63,0.042261332273483276
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,127,0.04367466767628988
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,fp8,127,0.04651733239491781
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,1,0.05259199937184652
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,fp8,1,0.055215999484062195
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,3,0.05299200117588043
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,fp8,7,0.055013333757718406
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,7,0.05297600229581197
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,fp8,3,0.055434669057528176
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,15,0.05309866865475973
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,fp8,15,0.05509866774082184
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,31,0.053360000252723694
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,fp8,31,0.05527999997138977
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,63,0.05646933118502299
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,fp8,63,0.0554720014333725
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,127,0.06256533165772755
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,fp8,127,0.06612266600131989
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,1,0.07893866797288258
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,fp8,1,0.08388266960779826
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,3,0.07977066437403361
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,fp8,3,0.0844053328037262
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,7,0.07945600152015686
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,fp8,7,0.08425600330034892
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,31,0.08407466610272725
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,fp8,31,0.08423999945322673
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,15,0.08019199967384338
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,fp8,15,0.0846720039844513
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,63,0.08770133058230083
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,fp8,63,0.08782399694124858
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,127,0.09046399593353271
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,fp8,127,0.10651733477910359
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,1,0.13085866967837015
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,fp8,1,0.13781866431236267
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,3,0.1307146648565928
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,7,0.13239999612172446
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,fp8,3,0.1390773355960846
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,fp8,7,0.13768000404040018
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,15,0.1368053356806437
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,fp8,15,0.13831466436386108
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,1,0.011077333241701126
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,31,0.13970133662223816
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,fp8,31,0.14087466398874918
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,3,0.011237333218256632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,1,0.011936000237862269
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,3,0.011952000359694162
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,63,0.14153599739074707
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,7,0.011120000233252844
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,7,0.011994666109482447
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,fp8,63,0.14593600233395895
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,15,0.01109333336353302
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,15,0.011946666985750198
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,31,0.012117333710193634
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,31,0.011125333607196808
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,63,0.011109333485364914
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,fp8,127,0.17733333508173624
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,127,0.14658666650454202
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,63,0.011909333368142446
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,127,0.011242666592200598
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,127,0.013327999661366144
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,255,0.012341332932313284
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,255,0.014671999961137772
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,511,0.015674666812022526
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,511,0.013776000589132309
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,2047,0.02295999974012375
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,1023,0.018197332819302876
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,2047,0.02038399999340375
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,1023,0.01602666700879733
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,4095,0.02903999884923299
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,4095,0.032773333291212715
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,8191,0.04774933556715647
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,8191,0.051882664362589516
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,16383,0.08342933654785156
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,3,0.012437333663304647
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,16383,0.0946613351504008
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,1,0.011301333705584208
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,1,0.012304000556468964
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,3,0.011429333438475927
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,7,0.011482667177915573
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,7,0.012191999703645706
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,15,0.011584000041087469
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,15,0.01240533341964086
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,31,0.011450666934251785
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,31,0.011952000359694162
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,63,0.01157333329319954
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,63,0.012223999947309494
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,255,0.015386667102575302
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,127,0.011413333316644033
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,127,0.013317332913478216
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,255,0.012784000486135483
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,511,0.014090667168299357
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,511,0.0164533331990242
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,1023,0.01657066618402799
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,1023,0.018592000007629395
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,2047,0.0205226664741834
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,8191,0.04934933284918467
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,2047,0.023455999791622162
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,4095,0.030389333764712017
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,4095,0.033173332611719765
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,8191,0.054192001620928444
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,16383,0.08488532900810242
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,16383,0.10030399759610494
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,1,0.012117333710193634
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,1,0.013002666334311167
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,3,0.011973333855470022
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,3,0.013023999830087027
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,7,0.01191466674208641
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,7,0.01292266696691513
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,15,0.01163200040658315
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,15,0.01333333303531011
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,31,0.011722666521867117
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,31,0.01331199953953425
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,63,0.011600000162919363
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,63,0.013471999516089758
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,127,0.012234666695197424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,127,0.014245333770910898
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,255,0.013712000101804733
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,511,0.014864000181357065
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,255,0.015493333339691162
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,511,0.016421332955360413
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,1023,0.016415999581416447
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,4095,0.03438399980465571
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,2047,0.0220320001244545
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,1023,0.019343999524911244
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,2047,0.023578666150569916
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,4095,0.03281066566705704
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,8191,0.05426133175690969
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,8191,0.057722667853037514
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,16383,0.09363733728726704
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,16383,0.10050666332244873
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,1,0.011786667009194693
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,1,0.012874666601419449
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,3,0.012186666329701742
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,3,0.014229333649079004
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,7,0.012005332857370377
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,7,0.012863999853531519
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,15,0.012400000045696894
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,15,0.013210666676362356
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,31,0.012047999848922094
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,31,0.01309866706530253
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,63,0.012063999970753988
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,63,0.013429333766301474
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,127,0.0122079998254776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,127,0.014357333381970724
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,255,0.013776000589132309
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,255,0.016250666230916977
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,511,0.01545599972208341
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,511,0.01807466646035512
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,1023,0.017301333447297413
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,1023,0.019472000499566395
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,2047,0.030346666773160298
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,2047,0.025237334271272022
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,4095,0.044682666659355164
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,4095,0.03907199949026108
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,8191,0.0763679991165797
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,8191,0.06073066592216492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,16383,0.1362986663977305
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,16383,0.10262933373451233
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,1,0.03853866706291834
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,fp8,1,0.03928533444801966
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,3,0.03873066604137421
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,fp8,3,0.03929600119590759
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,7,0.03894400099913279
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,fp8,7,0.03938666731119156
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,15,0.03898133337497711
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,fp8,15,0.0395413339138031
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,31,0.04091199984153112
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,fp8,31,0.039333333571751915
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,1,0.06502399841944377
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,63,0.04557333389918009
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,fp8,63,0.040789333482583366
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,fp8,1,0.06358399987220764
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,3,0.06506666541099548
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,fp8,3,0.06359466910362244
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,7,0.06520533561706543
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,15,0.06753600140412648
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,fp8,7,0.06332799792289734
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,fp8,15,0.06366933385531108
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,31,0.07212266822655995
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,fp8,31,0.0641599992911021
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,63,0.07361599802970886
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,fp8,63,0.07206933200359344
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,1,0.11617066462834676
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,fp8,1,0.11376532912254333
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,3,0.1158026655515035
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,fp8,3,0.11340799927711487
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,7,0.12119999527931213
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,fp8,7,0.1142026682694753
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,15,0.12788800398508707
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,fp8,15,0.11564800143241882
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,31,0.12921599547068277
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,1,0.2129653294881185
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,fp8,31,0.12449066837628682
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,63,0.12966400384902954
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,fp8,63,0.12687466541926065
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,fp8,1,0.20812799533208212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,3,0.21926399072011313
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,fp8,3,0.20736533403396606
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,7,0.23038933674494425
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,fp8,7,0.21127466360727945
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,15,0.23040000597635904
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,fp8,15,0.22055466969807944
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,31,0.23236799240112305
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,fp8,31,0.224400003751119
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,63,0.23427200317382812
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,fp8,63,0.2262773315111796
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,float16,1,0.07389333347479503
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,fp8,3,0.06748799979686737
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,fp8,1,0.0682239979505539
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,float16,3,0.07382933298746745
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,float16,7,0.07473066449165344
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,fp8,7,0.06828799843788147
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,fp8,15,0.06977599859237671
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,float16,15,0.07509333391984303
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,float16,31,0.07575466732184093
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,float16,1,0.12778133153915405
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,fp8,31,0.07547733187675476
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,fp8,1,0.11436800161997478
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,float16,7,0.13037332892417908
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,fp8,7,0.11980266372362773
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,float16,3,0.12966400384902954
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,fp8,3,0.11681066950162251
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,float16,15,0.1320319970448812
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,fp8,15,0.1253866652647654
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,float16,31,0.13134933511416116
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,float16,1,0.24004799127578735
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,fp8,1,0.2217386762301127
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,fp8,31,0.12713600198427835
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,float16,3,0.24162666002909342
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,fp8,3,0.22376533349355063
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,float16,7,0.24266133705774942
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,fp8,7,0.23141332467397055
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,fp8,15,0.23369600375493368
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,float16,15,0.24154667059580484
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,float16,31,0.2432746688524882
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,fp8,31,0.23458667596181235
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,float16,1,0.4474240144093831
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,fp8,1,0.4222773313522339
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,float16,3,0.4482986529668172
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,1,0.011477333803971609
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,fp8,3,0.4257440169652303
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,fp8,1,0.012346666306257248
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,float16,7,0.4480959971745809
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,3,0.011317333827416102
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,fp8,3,0.012234666695197424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,fp8,7,0.4261973301569621
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,float16,15,0.4490400155385335
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,7,0.01129066695769628
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,15,0.011509332805871964
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,fp8,7,0.012256000190973282
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,31,0.011450666934251785
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,fp8,31,0.012293333808581034
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,fp8,15,0.01221866657336553
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,fp8,63,0.012517333030700684
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,63,0.011551999797423681
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,127,0.011440000186363855
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,fp8,127,0.013728000223636627
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,255,0.012821332861979803
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,fp8,255,0.014287999520699183
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,511,0.014762666076421738
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,float16,31,0.45024534066518146
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,fp8,511,0.01714666684468587
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,fp8,31,0.43292800585428876
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,1023,0.019205333044131596
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,fp8,1023,0.02170666555563609
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,2047,0.027776000400384266
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,fp8,2047,0.03173333406448364
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,4095,0.04660266637802124
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,8191,0.08261333405971527
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,fp8,4095,0.05115200082461039
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,1,0.012005332857370377
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,fp8,1,0.012970666090647379
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,3,0.011674666156371435
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,15,0.011733333269755045
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,fp8,8191,0.09303466478983562
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,fp8,3,0.013082666943470636
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,7,0.011994666109482447
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,fp8,7,0.012853333105643591
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,fp8,15,0.012954667210578918
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,31,0.011989332735538483
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,fp8,31,0.013023999830087027
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,63,0.011834666132926941
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,fp8,63,0.01292266696691513
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,127,0.011936000237862269
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,fp8,127,0.014303999642531076
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,255,0.013093333691358566
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,fp8,255,0.01551466683546702
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,511,0.01545599972208341
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,fp8,511,0.017477333545684814
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,1023,0.01951466624935468
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,fp8,1023,0.022677332162857056
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,2047,0.028490667541821797
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,fp8,15,0.42850132783253986
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,4095,0.04808533191680908
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,fp8,2047,0.03252266595760981
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,fp8,4095,0.052416001756985985
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,8191,0.08343999584515889
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,fp8,8191,0.09915199875831604
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,1,0.011973333855470022
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,fp8,1,0.01357866699496905
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,fp8,3,0.013130666067202887
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,3,0.012080000092585882
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,7,0.012245333443085352
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,fp8,7,0.013253333667914072
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,15,0.011973333855470022
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,fp8,15,0.013237333546082178
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,fp8,63,0.01332266628742218
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,31,0.01209066684047381
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,fp8,31,0.013045333325862885
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,63,0.012015999605258306
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,127,0.01239466667175293
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,fp8,127,0.014533333480358124
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,255,0.013839999834696451
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,fp8,255,0.01525866612792015
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,511,0.015562667200962702
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,fp8,511,0.018133333573738735
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,1023,0.019802667200565338
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,fp8,1023,0.02256533255179723
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,2047,0.032986665765444435
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,fp8,2047,0.03316800047953924
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,4095,0.051258668303489685
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,fp8,4095,0.05737066765626272
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,8191,0.09004799524943034
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,fp8,8191,0.09892266988754272
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,fp8,1,0.013557333499193192
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,3,0.012485332787036896
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,1,0.01251199965675672
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,fp8,3,0.013183999806642532
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,7,0.012351999680201212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,fp8,7,0.01351999988158544
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,15,0.012234666695197424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,fp8,15,0.013424000392357508
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,31,0.012330666184425354
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,fp8,31,0.013434667140245438
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,63,0.012383999923865
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,fp8,63,0.013621332744757334
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,127,0.01268799975514412
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,fp8,127,0.015397333850463232
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,fp8,255,0.015925332903862
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,255,0.014352000008026758
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,1023,0.02898666759332021
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,fp8,511,0.018453333526849747
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,511,0.016303999970356624
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,fp8,1023,0.023306667804718018
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,2047,0.04413333535194397
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,fp8,2047,0.037920000652472176
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,4095,0.07399466633796692
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,8191,0.13315199812253317
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,fp8,4095,0.058335999647776283
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,fp8,8191,0.10086933771769206
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,1,0.012106666962305704
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,fp8,1,0.012560000022252401
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,3,0.012005332857370377
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,fp8,3,0.013450667262077332
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,7,0.011610666910807291
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,15,0.011781333635250727
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,fp8,7,0.012650666137536367
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,fp8,15,0.0129120002190272
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,31,0.012096000214417776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,fp8,31,0.012645332763592402
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,63,0.012117333710193634
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,fp8,63,0.012965332716703415
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,127,0.012042666474978128
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,fp8,127,0.01444799949725469
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,255,0.01341333364446958
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,fp8,255,0.015487999965747198
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,511,0.015493333339691162
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,fp8,511,0.0174346665541331
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,1023,0.019834666202465694
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,fp8,1023,0.023013333479563396
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,2047,0.02874133239189784
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,fp8,2047,0.03204799940188726
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,4095,0.04849066833655039
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,fp8,4095,0.05227733155091604
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,8191,0.08303999900817871
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,fp8,8191,0.099781334400177
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,1,0.012304000556468964
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,fp8,1,0.013434667140245438
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,3,0.011893333246310553
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,fp8,3,0.012896000097195307
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,7,0.012282667060693106
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,fp8,7,0.013466666142145792
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,15,0.011957333733638128
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,fp8,15,0.013056000073750814
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,31,0.01184533288081487
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,fp8,31,0.013349333157142004
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,63,0.01192533348997434
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,fp8,63,0.013445333888133367
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,127,0.012037333101034164
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,fp8,127,0.014778666198253632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,255,0.013605333864688873
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,fp8,255,0.015557333827018738
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,511,0.01602666700879733
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,fp8,511,0.017749333133300144
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,1023,0.020448000480731327
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,fp8,1023,0.02276266614596049
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,2047,0.03690666705369949
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,fp8,2047,0.033146666983763375
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,4095,0.053616002202034
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,fp8,4095,0.05653866628805796
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,8191,0.09335466225941975
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,fp8,8191,0.10108266274134318
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,3,0.01240533341964086
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,1,0.0124746672809124
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,fp8,1,0.013514666507641474
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,fp8,3,0.013546666751305262
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,7,0.012448000411192576
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,fp8,15,0.01333333303531011
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,fp8,7,0.013376000026861826
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,63,0.012682666381200155
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,15,0.013002666334311167
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,31,0.012661332885424295
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,fp8,31,0.013674666484196981
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,fp8,63,0.013610667238632837
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,127,0.013050666699806849
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,fp8,127,0.015077333897352219
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,255,0.014554666976133982
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,fp8,255,0.015765332927306492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,511,0.01701333373785019
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,fp8,511,0.018661333868900936
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,1023,0.028666667640209198
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,fp8,1023,0.023157333334287006
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,2047,0.043338666359583534
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,fp8,2047,0.036746665835380554
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,4095,0.07469866673151652
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,fp8,4095,0.05708266794681549
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,8191,0.13644267121950784
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,fp8,8191,0.09706133604049683
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,fp8,1,0.017957333475351334
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,1,0.016352000335852306
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,3,0.016410666207472484
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,fp8,3,0.01754666616519292
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,7,0.016565332810084026
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,fp8,7,0.017616000026464462
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,15,0.01685333376129468
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,fp8,15,0.017792000124851864
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,31,0.016730666160583496
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,fp8,31,0.017786666750907898
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,63,0.016672000288963318
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,fp8,63,0.017914666483799618
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,127,0.01720000058412552
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,fp8,127,0.02080533280968666
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,255,0.020586666961510975
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,fp8,255,0.022069332500298817
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,511,0.03030399978160858
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,fp8,511,0.0276053324341774
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,1023,0.046570668617884316
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,4095,0.13552000125249228
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,fp8,1023,0.04141866664091746
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,2047,0.07698133091131847
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,fp8,2047,0.06292800108591716
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,fp8,4095,0.10413866241772969
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,1,0.011146667102972666
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,1,0.012357333054145178
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,3,0.011130666981140772
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,8191,0.25142399470011395
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,fp8,8191,0.1890933314959208
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,3,0.011887999872366587
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,7,0.011242666592200598
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,7,0.011920000116030375
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,15,0.011343999455372492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,15,0.01249066616098086
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,31,0.011055999745925268
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,31,0.012362666428089142
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,63,0.011402666568756104
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,63,0.01219733307758967
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,127,0.011215999722480774
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,127,0.013466666142145792
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,255,0.012159999459981918
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,255,0.013471999516089758
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,511,0.01239466667175293
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,511,0.013850666582584381
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,1023,0.012975999464591345
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,1023,0.013861333330472311
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,2047,0.013951999445756277
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,2047,0.015061333775520325
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,4095,0.015130666395028433
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,4095,0.016261332978804905
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,8191,0.017509333789348602
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,8191,0.018944000204404194
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,16383,0.021920000513394673
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,16383,0.02443733314673106
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,32767,0.0322026660044988
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,32767,0.03455466777086258
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,1,0.01121066634853681
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,1,0.012042666474978128
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,3,0.011120000233252844
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,3,0.012432000289360682
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,7,0.011136000355084738
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,7,0.012341332932313284
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,15,0.011034666250149408
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,15,0.012421333541472753
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,31,0.011098666737476984
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,31,0.012378666549921036
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,63,0.011445333560307821
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,63,0.012085333466529846
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,127,0.012063999970753988
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,127,0.013072000195582708
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,255,0.012250666817029318
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,255,0.013232000172138214
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,511,0.012634667257467905
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,511,0.013823999712864557
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,1023,0.012874666601419449
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,1023,0.014442666123310724
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,2047,0.01423466702302297
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,2047,0.015861333658297855
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,4095,0.016106666376193363
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,4095,0.017322666943073273
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,8191,0.018357332795858383
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,8191,0.01952533299724261
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,16383,0.022858666876951855
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,16383,0.025066666305065155
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,32767,0.032885332902272545
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,7,0.011274666835864386
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,32767,0.03509866694609324
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,1,0.011445333560307821
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,1,0.012362666428089142
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,3,0.01137599969903628
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,3,0.012416000167528788
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,7,0.012138667205969492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,15,0.011557333171367645
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,15,0.012128000458081564
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,31,0.011530666301647821
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,31,0.012533333152532578
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,63,0.011584000041087469
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,63,0.011989332735538483
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,127,0.01156266654531161
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,127,0.013061333447694778
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,255,0.012437333663304647
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,255,0.013248000293970108
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,511,0.012624000509579977
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,511,0.014303999642531076
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,1023,0.013946666071812311
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,1023,0.015226667126019796
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,2047,0.01532799998919169
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,2047,0.016794666647911072
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,4095,0.016906666258970898
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,4095,0.017349333812793095
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,8191,0.018511999398469925
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,8191,0.020469332734743755
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,16383,0.023813332120577495
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,16383,0.025466665625572205
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,32767,0.035429333647092186
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,32767,0.03579733272393545
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,1,0.011231999844312668
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,1,0.012362666428089142
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,15,0.01138666644692421
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,3,0.011343999455372492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,3,0.012495999534924826
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,7,0.011253333340088526
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,7,0.01209066684047381
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,15,0.01232533281048139
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,31,0.011418666690587997
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,31,0.012319999436537424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,63,0.011477333803971609
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,63,0.0122079998254776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,127,0.011610666910807291
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,127,0.013663999736309052
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,255,0.012533333152532578
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,255,0.014090667168299357
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,511,0.013104000439246496
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,511,0.014917333920796713
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,1023,0.013872000078360239
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,1023,0.015509333461523056
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,2047,0.015423999478419622
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,2047,0.01748266691962878
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,4095,0.0164533331990242
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,4095,0.018277333428462345
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,8191,0.018917333334684372
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,8191,0.020495999604463577
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,16383,0.03146666785081228
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,16383,0.026602665583292644
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,32767,0.046821330984433494
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,32767,0.04029866556326548
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,1,0.010938666760921478
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,1,0.012122667084137598
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,3,0.011029332876205444
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,3,0.012351999680201212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,7,0.011120000233252844
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,7,0.012074666718641916
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,15,0.011237333218256632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,15,0.013503999759753546
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,31,0.01129066695769628
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,31,0.01240533341964086
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,63,0.011146667102972666
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,63,0.012389333297808966
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,127,0.011055999745925268
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,127,0.013487999637921652
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,255,0.012293333808581034
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,255,0.013306666165590286
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,511,0.012709333250919977
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,511,0.013679999858140945
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,1023,0.013823999712864557
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,1023,0.014778666198253632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,2047,0.014869333555301031
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,2047,0.015909332782030106
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,4095,0.017418666432301205
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,4095,0.018805333723624546
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,8191,0.020554666717847187
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,8191,0.024192000428835552
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,16383,0.029152000943819683
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,16383,0.03364799916744232
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,32767,0.047413334250450134
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,32767,0.052746668457984924
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,1,0.011391999820868174
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,7,0.011322667201360067
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,1,0.012191999703645706
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,3,0.011098666737476984
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,3,0.012069333344697952
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,7,0.012213333199421564
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,15,0.011493333925803503
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,15,0.012479999413092932
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,31,0.011461333682139715
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,31,0.012410666793584824
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,63,0.011482667177915573
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,63,0.012250666817029318
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,127,0.011264000087976456
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,127,0.013434667140245438
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,255,0.012346666306257248
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,255,0.013594667116800943
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,511,0.012853333105643591
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,2047,0.016586666305859882
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,4095,0.018031999468803406
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,1023,0.014538666854302088
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,511,0.014426667243242264
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,1023,0.01581866666674614
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,2047,0.015962666521469753
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,4095,0.019551999866962433
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,8191,0.02117866774400075
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,8191,0.023989332218964893
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,16383,0.030165334542592365
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,16383,0.03446399917205175
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,32767,0.04993600149949392
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,32767,0.05625066657861074
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,1,0.011642667154471079
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,1,0.012495999534924826
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,3,0.011605333536863327
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,3,0.012261333564917246
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,7,0.011503999431928
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,7,0.01227733368674914
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,15,0.011727999895811081
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,15,0.012154666086037954
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,31,0.011535999675591787
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,31,0.012602667013804117
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,63,0.01163200040658315
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,63,0.012373333175977072
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,127,0.011488000551859537
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,127,0.01331199953953425
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,255,0.013221333424250284
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,255,0.013978666315476099
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,511,0.014021333307027817
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,511,0.014746667196353277
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,1023,0.015589332828919092
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,1023,0.015706667055686314
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,2047,0.01621333385507266
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,2047,0.016751999656359356
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,4095,0.01836799954374631
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,4095,0.019215999792019527
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,32767,0.054085334142049156
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,8191,0.021456000705560047
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,8191,0.024485332270463307
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,16383,0.03416533271471659
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,16383,0.03532800078392029
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,32767,0.05683733522891998
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,1,0.012693333129088083
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,1,0.01192533348997434
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,3,0.011418666690587997
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,3,0.01258133351802826
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,7,0.011770666887362799
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,7,0.012613333761692047
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,15,0.011701333026091257
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,15,0.012645332763592402
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,31,0.011626667032639185
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,31,0.012752000242471695
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,63,0.011823999385039011
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,63,0.012479999413092932
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,127,0.012063999970753988
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,127,0.01414399966597557
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,255,0.013290667285521826
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,255,0.014767999450365702
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,511,0.013536000003417334
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,511,0.014709333578745524
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,1023,0.015482666591803232
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,1023,0.01646399994691213
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,2047,0.01670933390657107
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,2047,0.017322666943073273
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,4095,0.019141333798567455
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,4095,0.0199946661790212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,8191,0.030981334547201794
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,8191,0.02474133421977361
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,16383,0.0455626646677653
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,16383,0.04053333401679993
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,32767,0.07623999814192454
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,32767,0.060090666015942894
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,1,0.012608000387748083
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,fp8,1,0.013306666165590286
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,3,0.012810666114091873
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,fp8,3,0.013264000415802002
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,7,0.012944000462690989
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,fp8,7,0.01340266689658165
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,15,0.012714666624863943
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,fp8,15,0.013728000223636627
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,31,0.013045333325862885
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,fp8,31,0.013877333452304205
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,63,0.012784000486135483
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,fp8,63,0.013530666629473368
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,127,0.012768000364303589
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,fp8,127,0.01534933348496755
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,255,0.014789332946141561
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,fp8,255,0.01595199977358182
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,511,0.017637333522240322
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,fp8,511,0.017946666727463405
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,fp8,1023,0.0232640008131663
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,1023,0.021173333128293354
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,2047,0.038592000802357994
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,1,0.013151999562978745
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,fp8,1,0.014629332969586054
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,fp8,2047,0.03333866596221924
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,3,0.013114667187134424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,7,0.01333333303531011
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,fp8,3,0.014303999642531076
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,fp8,7,0.014458666245142618
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,15,0.013493333011865616
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,fp8,4095,0.05804799993832906
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,4095,0.05398933092753092
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,31,0.013306666165590286
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,fp8,15,0.014352000008026758
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,fp8,31,0.014149333039919535
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,63,0.013199999928474426
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,fp8,127,0.01573866605758667
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,255,0.015114666273196539
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,fp8,63,0.01451733335852623
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,127,0.013359999905029932
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,fp8,255,0.016789333273967106
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,511,0.017173333714405697
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,fp8,511,0.01889066646496455
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,1023,0.029887999097506206
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,4095,0.0753653347492218
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,fp8,1023,0.024447999894618988
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,2047,0.04555733501911163
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,fp8,2047,0.03865066667397817
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,fp8,4095,0.05952533086140951
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,1,0.017152000218629837
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,fp8,1,0.01820266619324684
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,3,0.01720533271630605
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,fp8,3,0.0176959993938605
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,15,0.01724799970785777
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,7,0.016970666746298473
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,fp8,7,0.017797333498795826
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,fp8,15,0.018090666582187016
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,31,0.017184000462293625
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,fp8,31,0.018277333428462345
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,63,0.01749333366751671
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,fp8,63,0.017749333133300144
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,127,0.017504000415404636
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,fp8,127,0.020746666938066483
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,255,0.020928000410397846
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,fp8,255,0.022319999833901722
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,511,0.03158933420976003
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,fp8,511,0.028959999481836956
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,1023,0.04781866570313772
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,fp8,1023,0.04102399945259094
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,2047,0.07774933179219563
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,fp8,2047,0.06195733447869619
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,4095,0.135535995165507
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,fp8,4095,0.10191466410954793
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,fp8,3,0.025514667232831318
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,1,0.02402666707833608
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,fp8,1,0.025589334468046825
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,3,0.02386666586001714
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,7,0.023887999355793
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,fp8,7,0.025472000241279602
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,15,0.024085332949956257
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,fp8,15,0.02571733295917511
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,31,0.024559999505678814
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,fp8,31,0.025455998877684276
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,63,0.023999998966852825
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,fp8,63,0.025616000096003216
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,127,0.024826665719350178
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,fp8,127,0.030159999926884968
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,255,0.03692266593376795
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,fp8,255,0.03439466655254364
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,511,0.05095999936262766
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,fp8,511,0.04933866858482361
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,1023,0.08071466783682506
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,fp8,1023,0.0693280001481374
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,2047,0.13804800311724344
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,fp8,2047,0.11252267162005107
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,1,0.010629333555698395
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,1,0.011626667032639185
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,3,0.01062400018175443
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,4095,0.25446399052937824
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,fp8,4095,0.1974239945411682
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,3,0.011727999895811081
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,7,0.010693332801262537
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,7,0.011706666400035223
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,15,0.010757333288590113
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,15,0.011802667131026586
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,31,0.011247999966144562
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,31,0.011578666667143503
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,63,0.010778666784365972
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,63,0.011690666278203329
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,127,0.0106133334338665
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,127,0.013072000195582708
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,255,0.012069333344697952
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,255,0.013424000392357508
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,511,0.01314666618903478
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,511,0.014314666390419006
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,1023,0.014378666877746582
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,1023,0.015184000134468079
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,2047,0.015829333414634068
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,2047,0.01815466706951459
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,4095,0.02045866722861926
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,4095,0.022944000860055287
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,8191,0.02899733434120814
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,8191,0.03294933338960012
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,16383,0.047055999437967934
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,16383,0.05202133456865946
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,32767,0.08362133304278056
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,32767,0.09061866998672485
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,1,0.011183999478816986
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,1,0.011999999483426413
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,3,0.01099733387430509
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,3,0.011749333391586939
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,7,0.011007999380429586
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,7,0.011722666521867117
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,15,0.010970667004585266
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,15,0.01173866664369901
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,31,0.011183999478816986
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,31,0.012128000458081564
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,63,0.011114666859308878
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,63,0.011813333878914515
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,127,0.011039999624093374
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,127,0.013269333789745966
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,255,0.012298667182525
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,255,0.014117332796255747
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,511,0.013760000467300415
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,511,0.015743999431530636
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,1023,0.015573333948850632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,1023,0.01657066618402799
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,2047,0.01629866659641266
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,8191,0.03350933392842611
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,2047,0.018698666244745255
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,4095,0.020879998803138733
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,4095,0.023376000424226124
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,8191,0.030981334547201794
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,16383,0.04993066688378652
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,16383,0.055642664432525635
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,32767,0.084714670976003
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,32767,0.10116799672444661
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,1,0.011418666690587997
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,1,0.011690666278203329
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,3,0.01137599969903628
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,15,0.012191999703645706
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,3,0.012293333808581034
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,7,0.011418666690587997
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,7,0.012159999459981918
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,15,0.011594666788975397
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,31,0.01145600030819575
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,31,0.01190399999419848
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,63,0.011343999455372492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,63,0.012069333344697952
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,127,0.011760000139474869
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,127,0.01370666672786077
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,255,0.013056000073750814
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,255,0.01394133393963178
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,511,0.015200000256299973
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,511,0.015189333508412043
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,1023,0.01573866605758667
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,1023,0.016165333489576977
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,2047,0.01693333312869072
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,2047,0.018933333456516266
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,4095,0.021349333226680756
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,4095,0.02365333338578542
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,8191,0.03731200098991394
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,8191,0.03348266581694285
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,16383,0.053269331653912864
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,16383,0.05569066603978475
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,32767,0.10709333419799805
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,32767,0.09627733627955119
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,1,0.01192533348997434
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,1,0.013221333424250284
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,3,0.012074666718641916
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,3,0.012773333738247553
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,7,0.011957333733638128
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,7,0.013429333766301474
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,15,0.011781333635250727
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,15,0.01293333371480306
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,31,0.011999999483426413
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,127,0.014266667266686758
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,31,0.012896000097195307
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,63,0.011749333391586939
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,63,0.013077333569526672
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,127,0.012191999703645706
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,255,0.01357866699496905
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,255,0.014671999961137772
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,511,0.014416000495354334
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,511,0.016250666230916977
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,1023,0.01591466615597407
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,1023,0.016800000021855038
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,2047,0.017242666333913803
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,2047,0.01940800001223882
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,4095,0.0303413321574529
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,4095,0.024442667762438457
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,8191,0.04552533229192098
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,8191,0.040149333576361336
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,16383,0.0759093314409256
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,16383,0.062165334820747375
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,32767,0.13563733299573263
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,32767,0.10683733224868774
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,1,0.014629332969586054
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,fp8,1,0.01581866666674614
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,3,0.014335999886194864
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,fp8,3,0.015509333461523056
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,7,0.014442666123310724
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,fp8,15,0.015664000064134598
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,fp8,7,0.015658666690190632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,15,0.014511999984582266
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,31,0.014479999740918478
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,fp8,31,0.015717333803574245
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,63,0.014463999619086584
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,fp8,127,0.01721599946419398
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,fp8,63,0.015562667200962702
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,127,0.014730667074521383
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,255,0.016373333831628162
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,fp8,255,0.018021332720915478
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,511,0.018746666610240936
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,fp8,511,0.020629333953062694
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,1023,0.030741333961486816
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,fp8,1023,0.025514667232831318
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,fp8,2047,0.040405333042144775
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,2047,0.045647998650868736
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,1,0.017818666994571686
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,fp8,1,0.019760000209013622
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,3,0.017893332988023758
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,fp8,3,0.019573333362738293
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,7,0.017909333109855652
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,fp8,7,0.019365333020687103
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,15,0.017909333109855652
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,31,0.018245333184798557
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,63,0.01800000046690305
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,fp8,15,0.01933866615096728
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,fp8,31,0.01966933285196622
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,fp8,63,0.019727999965349834
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,127,0.018458666900793713
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,fp8,127,0.022053333620230358
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,255,0.02160533269246419
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,fp8,255,0.023605334262053173
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,fp8,511,0.02937600016593933
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,511,0.03440000116825104
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,1023,0.04924266537030538
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,fp8,1023,0.04404266675313314
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,1,0.02447466552257538
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,fp8,1,0.026485333840052288
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,3,0.024549332757790882
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,2047,0.0788213312625885
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,fp8,2047,0.06566399832566579
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,15,0.024869332710901897
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,fp8,15,0.026496000587940216
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,31,0.02481599897146225
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,fp8,3,0.02629333237806956
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,fp8,7,0.026149332523345947
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,fp8,63,0.026170666019121807
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,7,0.024325333535671234
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,127,0.02606933315594991
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,fp8,127,0.031040000418821972
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,63,0.02475733309984207
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,255,0.03851733356714249
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,fp8,255,0.03508266558249792
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,fp8,31,0.026250667870044708
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,511,0.052389333645502724
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,2047,0.1393173336982727
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,fp8,511,0.049786667029062905
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,fp8,2047,0.1113973359266917
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,fp8,1023,0.06986133257548015
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,1,0.037818667789300285
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,fp8,1,0.04080000023047129
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,1023,0.0817440003156662
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,fp8,3,0.040618665516376495
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,3,0.03752533346414566
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,7,0.03772799919048945
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,fp8,7,0.04062400013208389
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,fp8,15,0.041002665956815086
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,15,0.03789333254098892
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,31,0.03818133225043615
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,fp8,31,0.04088533421357473
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,63,0.03878399978081385
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,fp8,63,0.040762667854626976
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,127,0.04452266792456309
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,fp8,127,0.05055999755859375
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,255,0.059658666451772056
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,fp8,255,0.06198399762312571
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,1023,0.14678933223088583
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,511,0.08868267138799031
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,fp8,511,0.08396266897519429
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,fp8,1023,0.1246506671110789
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,2047,0.2601226568222046
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,fp8,2047,0.21039466063181558
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,1,0.02181866765022278
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,fp8,1,0.023578666150569916
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,7,0.021984001000722248
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,3,0.021802666286627453
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,fp8,7,0.023589332898457844
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,15,0.021701333423455555
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,fp8,3,0.02347733328739802
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,fp8,15,0.02333866556485494
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,31,0.021744000415007275
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,fp8,31,0.023546665906906128
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,63,0.02189333240191142
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,fp8,63,0.023728000621000927
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,127,0.022346665461858112
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,fp8,127,0.026101333399613697
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,255,0.02605333427588145
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,fp8,255,0.028149334092934925
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,1,0.028330666323502857
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,511,0.038746667404969536
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,fp8,511,0.03580799947182337
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,fp8,1,0.030266667405764263
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,3,0.02815466622511546
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,fp8,3,0.030261332790056866
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,7,0.028181334336598713
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,fp8,7,0.03073066721359889
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,15,0.02869333326816559
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,fp8,15,0.030394665896892548
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,31,0.028437333802382152
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,fp8,31,0.030165334542592365
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,fp8,63,0.030117332935333252
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,63,0.028543998797734577
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,fp8,127,0.0358240008354187
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,127,0.029711998999118805
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,255,0.043247997760772705
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,fp8,255,0.04072533299525579
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,511,0.05761066575845083
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,fp8,511,0.0551093320051829
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,1,0.04102933406829834
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,fp8,1,0.0433599998553594
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,3,0.0413973331451416
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,fp8,3,0.04320533573627472
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,7,0.04102933406829834
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,fp8,7,0.04373333354791006
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,31,0.0415040006240209
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,fp8,31,0.04343999922275543
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,15,0.0414986660083135
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,fp8,15,0.043354665239652
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,63,0.04265599946180979
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,fp8,63,0.043466667334238686
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,127,0.04877866804599762
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,fp8,127,0.054757331808408104
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,255,0.061792001128196716
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,fp8,255,0.06467199822266896
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,3,0.0675786683956782
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,1,0.06738666693369548
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,fp8,1,0.07180266578992207
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,511,0.09257599711418152
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,fp8,3,0.07160000006357829
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,fp8,511,0.08729066451390584
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,15,0.06764266888300578
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,7,0.06735999882221222
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,fp8,15,0.07215466598669688
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,31,0.06853866577148438
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,fp8,7,0.07148266832033794
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,63,0.07332799832026164
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,127,0.07712533573309581
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,fp8,127,0.09341333309809367
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,fp8,63,0.07356800138950348
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,fp8,31,0.07188799977302551
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,255,0.1027893324693044
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,fp8,255,0.11006933450698853
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,1,0.039162665605545044
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,fp8,1,0.04139200101296107
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,3,0.03905600061019262
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,fp8,511,0.15547733505566916
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,511,0.16368533174196878
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,fp8,3,0.041690667470296226
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,7,0.03882133215665817
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,15,0.03905600061019262
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,fp8,15,0.04155733436346054
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,fp8,7,0.041402667760849
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,31,0.03933866570393244
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,fp8,63,0.04197866717974345
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,fp8,31,0.04083733260631561
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,63,0.039701332648595176
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,127,0.041834667325019836
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,fp8,127,0.04549333453178406
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,255,0.05330666899681091
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,fp8,255,0.05067733426888784
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,1,0.051221330960591636
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,fp8,1,0.054330666859944664
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,3,0.050800000627835594
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,fp8,3,0.05483733117580414
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,7,0.05143466591835022
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,fp8,7,0.05449600021044413
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,15,0.051301335295041404
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,fp8,15,0.05426666637261709
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,31,0.05162666738033295
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,fp8,31,0.054469332098960876
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,63,0.052890668312708534
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,fp8,63,0.054666668176651
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,127,0.060346667965253196
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,fp8,127,0.0647626668214798
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,255,0.07564799984296162
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,1,0.07715733349323273
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,fp8,255,0.07853333155314128
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,fp8,1,0.08097066481908162
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,3,0.07804266611735027
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,fp8,3,0.08154666423797607
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,7,0.07705600063006084
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,fp8,7,0.08150933186213176
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,15,0.07803733150164287
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,fp8,15,0.08087466657161713
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,31,0.08009600142637889
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,fp8,31,0.08152000109354655
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,63,0.0832586685816447
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,fp8,63,0.0842186709245046
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,127,0.08724266290664673
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,fp8,127,0.1037600040435791
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,255,0.11145599683125813
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,fp8,255,0.11821867028872173
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,1,0.12920533617337546
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,fp8,1,0.13634133338928223
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,3,0.12965866923332214
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,fp8,3,0.136954665184021
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,7,0.12981333335240683
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,fp8,7,0.13606933752695718
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,15,0.13127999504407248
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,31,0.1343839963277181
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,fp8,31,0.13876799742380777
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,fp8,15,0.13640000422795615
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,63,0.13732799887657166
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,fp8,63,0.1442026694615682
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,1,0.011242666592200598
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,1,0.011968000481526056
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,7,0.011285333583752314
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,127,0.14353600144386292
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,7,0.012015999605258306
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,fp8,127,0.175818661848704
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,15,0.01110400011142095
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,3,0.01118933285276095
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,3,0.01191466674208641
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,31,0.010981333752473196
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,63,0.011039999624093374
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,63,0.0124746672809124
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,127,0.01102399950226148
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,127,0.01310933381319046
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,15,0.012047999848922094
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,255,0.012266666938861212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,31,0.01201066623131434
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,255,0.0144213338692983
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,511,0.014064000298579534
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,511,0.01553600033124288
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,1023,0.01590399940808614
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,255,0.1894773244857788
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,1023,0.01785600061217944
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,2047,0.020053333292404812
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,fp8,255,0.2056586742401123
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,2047,0.02274133265018463
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,4095,0.02890666574239731
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,4095,0.03233066697915395
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,8191,0.047584002216657005
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,16383,0.08307733138402303
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,16383,0.09500799576441447
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,32767,0.1848586599032084
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,1,0.011429333438475927
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,1,0.012106666962305704
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,8191,0.05216533442338308
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,3,0.011418666690587997
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,3,0.012341332932313284
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,7,0.011301333705584208
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,31,0.012400000045696894
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,7,0.012266666938861212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,32767,0.15370133519172668
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,63,0.012266666938861212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,15,0.011370666325092316
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,15,0.012250666817029318
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,31,0.011525332927703857
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,63,0.011296000331640244
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,127,0.011407999942700068
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,127,0.013536000003417334
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,255,0.012837332983811697
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,255,0.016058667252461117
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,511,0.014698666830857595
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,511,0.015882667154073715
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,1023,0.01657066618402799
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,1023,0.018650667121013004
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,2047,0.020768000433842342
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,2047,0.023247999449570973
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,4095,0.029552000264326733
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,4095,0.033733333150545754
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,8191,0.049173335234324135
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,8191,0.05328533550103506
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,16383,0.0837653378645579
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,16383,0.10026666522026062
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,32767,0.155349334081014
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,32767,0.18406933546066284
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,1,0.012058666596810022
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,1,0.012416000167528788
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,3,0.011866666376590729
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,3,0.012554666648308435
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,7,0.011727999895811081
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,7,0.012784000486135483
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,15,0.012106666962305704
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,15,0.012714666624863943
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,31,0.011978667229413986
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,31,0.0124746672809124
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,63,0.012063999970753988
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,63,0.012560000022252401
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,127,0.012266666938861212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,127,0.013999999811251959
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,255,0.013647999614477158
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,255,0.015125333021084467
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,511,0.014837333311637243
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,511,0.016042667130629223
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,1023,0.01657066618402799
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,1023,0.01883200059334437
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,2047,0.02164799968401591
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,2047,0.02348266790310542
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,4095,0.03651199986537298
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,4095,0.03416533271471659
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,8191,0.05301333467165629
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,8191,0.055029332637786865
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,16383,0.09142399827639262
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,16383,0.09556800127029419
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,32767,0.17342400550842285
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,32767,0.1745013395945231
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,1,0.01210133358836174
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,1,0.013061333447694778
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,3,0.011765333513418833
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,3,0.012954667210578918
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,7,0.011765333513418833
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,31,0.013023999830087027
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,7,0.013269333789745966
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,15,0.011594666788975397
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,15,0.013072000195582708
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,31,0.012144000579913458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,63,0.011839999506870905
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,63,0.013445333888133367
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,127,0.01211200033624967
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,127,0.014426667243242264
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,255,0.01379199946920077
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,255,0.015957333147525787
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,511,0.015333333363135656
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,511,0.01727466657757759
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,1023,0.01718933383623759
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,1023,0.01931200052301089
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,2047,0.03047466774781545
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,2047,0.025568000972270966
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,4095,0.04534400006135305
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,4095,0.038346665600935616
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,8191,0.0757066657145818
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,8191,0.06106133262316386
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,16383,0.13593600193659464
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,16383,0.101583997408549
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,32767,0.25909332434336346
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,32767,0.18567466735839844
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,fp8,1,0.039317332208156586
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,1,0.038634667793909706
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,3,0.03871466716130575
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,fp8,3,0.039434666434923805
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,7,0.03851733356714249
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,15,0.03955733279387156
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,fp8,7,0.039120001097520195
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,fp8,15,0.039503999054431915
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,31,0.03908266623814901
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,fp8,31,0.0391893337170283
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,63,0.04463466505209605
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,fp8,63,0.039733332892258964
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,127,0.046495998899141945
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,fp8,127,0.05225066840648651
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,fp8,1,0.06318399806817372
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,1,0.06477866570154826
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,3,0.06468266745408376
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,fp8,3,0.0633653352657954
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,7,0.06497600177923839
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,fp8,7,0.063701331615448
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,31,0.07108800113201141
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,15,0.06603200236956279
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,fp8,31,0.06361599763234456
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,fp8,15,0.06334400177001953
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,63,0.07230933507283528
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,127,0.07575466732184093
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,1,0.1111253301302592
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,fp8,1,0.11274666587511699
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,fp8,63,0.07082133491834004
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,fp8,127,0.09011733531951904
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,3,0.11081066727638245
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,fp8,3,0.11297600467999776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,7,0.11204800009727478
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,fp8,7,0.11187733213106792
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,15,0.11674132943153381
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,fp8,15,0.11293866237004598
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,31,0.121888001759847
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,fp8,31,0.11746666828791301
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,63,0.12456533312797546
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,1,0.21234132846196493
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,fp8,63,0.12289599577585857
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,fp8,1,0.20735466480255127
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,fp8,127,0.15281066298484802
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,127,0.13142399986584982
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,3,0.21579732497533163
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,fp8,3,0.20549333095550537
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,7,0.22665067513783774
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,fp8,7,0.20973867177963257
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,15,0.23056532939275107
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,fp8,15,0.21957866350809732
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,31,0.22933334112167358
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,fp8,31,0.22263999780019125
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,63,0.23455999294916788
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,fp8,63,0.22390933831532797
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,127,0.2419253389040629
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,fp8,127,0.2927573323249817
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,1,0.06817600131034851
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,fp8,1,0.06680533289909363
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,fp8,3,0.06674133241176605
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,3,0.06835733354091644
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,7,0.07149333258469899
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,fp8,7,0.06673599779605865
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,15,0.07384000221888225
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,fp8,15,0.06726400057474773
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,31,0.07551999886830647
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,fp8,31,0.07106666763623555
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,63,0.07602666815121968
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,fp8,63,0.07718400160471599
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,1,0.12229333321253459
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,fp8,1,0.11353600025177002
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,3,0.1253866652647654
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,fp8,3,0.11447466413180034
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,7,0.12917332847913107
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,fp8,7,0.11529067158699036
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,15,0.1301759978135427
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,fp8,15,0.119759996732076
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,31,0.1316373348236084
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,fp8,31,0.125408003727595
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,63,0.13179733355840048
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,fp8,63,0.12802666425704956
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,1,0.21222400665283203
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,fp8,1,0.21270400285720825
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,3,0.21252266565958658
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,fp8,3,0.21221333742141724
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,fp8,7,0.21404266357421875
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,7,0.22021865844726562
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,15,0.22521066665649414
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,fp8,15,0.2239840030670166
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,fp8,31,0.22753065824508667
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,63,0.23449599742889404
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,fp8,63,0.23056532939275107
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,1,0.4395253260930379
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,fp8,1,0.4095360040664673
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,3,0.44386132558186847
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,fp8,3,0.42108798027038574
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,31,0.229695995648702
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,7,0.4448586702346802
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,fp8,7,0.42658666769663495
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,15,0.4434346755345662
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,1,0.011509332805871964
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,1,0.012442667037248611
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,fp8,15,0.4252053499221802
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,3,0.011754666765530905
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,3,0.012351999680201212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,7,0.011333333949247995
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,31,0.44973333676656085
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,15,0.011717333147923151
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,15,0.012453333785136541
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,31,0.011493333925803503
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,7,0.012223999947309494
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,fp8,31,0.43356800079345703
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,31,0.01250133290886879
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,63,0.012378666549921036
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,127,0.011247999966144562
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,127,0.01351999988158544
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,63,0.011722666521867117
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,255,0.012853333105643591
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,511,0.014736000448465347
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,511,0.017018667111794155
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,1023,0.01918399954835574
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,1023,0.02186666677395503
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,2047,0.028069332242012024
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,255,0.014368000129858652
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,2047,0.03151999910672506
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,63,0.45147732893625897
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,4095,0.05077866713205973
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,16383,0.15253866712252298
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,4095,0.04635733366012573
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,8191,0.08270399769147237
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,8191,0.09338666995366414
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,fp8,63,0.4333440065383911
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,1,0.012863999853531519
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,16383,0.1819146672884623
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,3,0.01321600005030632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,7,0.011733333269755045
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,7,0.012794667234023413
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,3,0.011546666423479715
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,15,0.012762666990359625
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,31,0.011994666109482447
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,31,0.012805332740147909
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,63,0.012842666357755661
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,63,0.011765333513418833
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,127,0.01181866725285848
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,127,0.014442666123310724
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,255,0.013317332913478216
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,511,0.015376000354687372
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,255,0.015061333775520325
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,511,0.017701332767804463
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,1023,0.019882666567961376
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,1023,0.022117334107557934
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,4095,0.04864533245563507
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,2047,0.028512001037597656
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,2047,0.032325332363446556
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,4095,0.05301866432030996
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,8191,0.08346666892369588
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,8191,0.09861866633097331
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,1,0.012426666915416718
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,16383,0.1530133287111918
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,16383,0.18331732352574667
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,1,0.013077333569526672
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,3,0.012085333466529846
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,3,0.013354666531085968
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,15,0.011744000017642975
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,1,0.01184533288081487
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,7,0.011754666765530905
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,7,0.0129120002190272
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,15,0.012138667205969492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,15,0.012896000097195307
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,31,0.012272000312805176
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,31,0.013157332936922709
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,255,0.014010666559139887
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,63,0.012234666695197424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,63,0.012917333592971167
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,127,0.012367999802033106
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,127,0.01482133318980535
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,255,0.015002666662136713
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,2047,0.036101333796978
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,511,0.015989333391189575
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,511,0.01823466643691063
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,1023,0.02057066683967908
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,1023,0.022175999979178112
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,2047,0.03246400008598963
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,4095,0.04925866425037384
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,4095,0.05481599768002828
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,8191,0.09226133426030476
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,8191,0.09514666597048442
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,16383,0.17191465695699057
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,16383,0.17479999860127768
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,1,0.012410666793584824
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,1,0.013466666142145792
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,3,0.013317332913478216
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,15,0.013573333621025085
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,3,0.013418667018413544
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,7,0.012261333564917246
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,7,0.013471999516089758
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,15,0.012330666184425354
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,31,0.012298667182525
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,31,0.013199999928474426
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,63,0.012362666428089142
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,63,0.013658666362365087
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,127,0.012453333785136541
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,127,0.015072000523408255
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,255,0.014373333503802618
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,255,0.01600533351302147
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,511,0.016117333124081295
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,511,0.01820266619324684
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,1023,0.02863466739654541
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,1023,0.024346667031447094
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,2047,0.04348266621430715
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,2047,0.0377813329299291
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,4095,0.07457600037256877
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,4095,0.059008002281188965
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,8191,0.13264532883961996
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,8191,0.10009066263834636
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,16383,0.2561653256416321
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,fp8,1,0.013167999684810638
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,16383,0.18531733751296997
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,1,0.0120319997270902
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,3,0.01221866657336553
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,fp8,3,0.013557333499193192
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,7,0.011909333368142446
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,fp8,7,0.01314666618903478
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,15,0.012058666596810022
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,fp8,15,0.012784000486135483
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,31,0.011968000481526056
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,fp8,31,0.013194666554530462
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,63,0.012170666207869848
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,fp8,63,0.013327999661366144
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,127,0.011994666109482447
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,fp8,127,0.014101333916187286
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,255,0.01349866638580958
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,fp8,255,0.014874666929244995
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,511,0.01523200049996376
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,fp8,511,0.01773333301146825
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,1023,0.01972266659140587
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,fp8,1023,0.02253866692384084
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,2047,0.028677334388097126
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,fp8,2047,0.03234133372704188
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,4095,0.04801600178082784
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,fp8,4095,0.05239999790986379
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,8191,0.08270399769147237
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,fp8,8191,0.09996267159779866
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,1,0.012005332857370377
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,fp8,1,0.012901333471139273
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,7,0.011866666376590729
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,3,0.012181332955757776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,fp8,3,0.012901333471139273
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,fp8,7,0.012858666479587555
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,15,0.011952000359694162
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,fp8,15,0.013461332768201828
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,31,0.011989332735538483
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,fp8,31,0.012986666212479273
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,63,0.011877333124478659
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,fp8,63,0.013376000026861826
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,127,0.012437333663304647
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,fp8,127,0.015119999647140503
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,255,0.013888000200192133
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,fp8,255,0.015578666081031164
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,511,0.015829333414634068
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,fp8,511,0.017845333864291508
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,1023,0.020090666910012562
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,fp8,1023,0.02294933299223582
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,2047,0.03628266602754593
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,fp8,2047,0.033386667569478355
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,4095,0.051781331499417625
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,fp8,4095,0.056976000467936196
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,8191,0.09372267127037048
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,fp8,8191,0.10016533732414246
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,1,0.012421333541472753
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,fp8,7,0.01350933313369751
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,fp8,1,0.013776000589132309
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,3,0.012543999900420507
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,31,0.012410666793584824
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,fp8,3,0.013466666142145792
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,7,0.012261333564917246
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,15,0.012453333785136541
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,fp8,15,0.013642666240533194
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,fp8,31,0.013584000368913015
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,63,0.012693333129088083
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,fp8,63,0.013679999858140945
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,127,0.012821332861979803
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,fp8,127,0.015013333410024643
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,255,0.014181333283583323
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,fp8,255,0.015925332903862
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,511,0.01657066618402799
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,fp8,511,0.018464000274737675
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,1023,0.028517333169778187
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,fp8,1023,0.023381332556406658
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,2047,0.04466133316357931
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,fp8,2047,0.038191998998324074
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,4095,0.07400533556938171
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,fp8,4095,0.058506667613983154
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,8191,0.13492266337076822
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,fp8,8191,0.10113599896430969
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,1,0.0164533331990242
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,fp8,1,0.017946666727463405
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,3,0.016682667036851246
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,fp8,3,0.01749333366751671
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,7,0.016293333222468693
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,fp8,7,0.017450666675964992
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,15,0.016586666305859882
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,fp8,15,0.01786133274435997
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,31,0.016586666305859882
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,fp8,31,0.017664000391960144
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,63,0.016538667182127636
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,fp8,63,0.017898666361967724
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,127,0.016522667060295742
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,fp8,127,0.020693333198626835
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,255,0.020453333854675293
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,fp8,255,0.02203733225663503
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,511,0.030591999491055805
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,fp8,511,0.02743999908367793
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,1023,0.04645866652329763
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,fp8,1023,0.04159999887148539
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,2047,0.0763679991165797
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,fp8,2047,0.06329066554705302
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,4095,0.1356106698513031
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,fp8,4095,0.10426666339238484
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,1,0.010949333508809408
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,1,0.012202666451533636
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,7,0.011029332876205444
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,8191,0.2519306739171346
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,fp8,8191,0.18831467628479004
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,3,0.011066666493813196
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,3,0.011999999483426413
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,7,0.012133333832025528
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,63,0.011407999942700068
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,15,0.01108266661564509
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,15,0.01221866657336553
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,31,0.011130666981140772
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,31,0.012442667037248611
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,63,0.012389333297808966
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,127,0.011120000233252844
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,127,0.013343999783198038
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,255,0.01219733307758967
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,255,0.013167999684810638
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,511,0.01251199965675672
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,511,0.013674666484196981
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,1023,0.012645332763592402
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,1023,0.014032000054915747
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,2047,0.013909333695967993
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,2047,0.015119999647140503
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,4095,0.01522133375207583
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,4095,0.016330666840076447
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,8191,0.017573333034912746
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,8191,0.01916266605257988
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,16383,0.022170667846997578
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,16383,0.02460266649723053
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,32767,0.031701333820819855
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,32767,0.03478399912516276
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,1,0.011061333119869232
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,7,0.012373333175977072
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,1,0.012170666207869848
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,3,0.010960000256697336
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,3,0.012122667084137598
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,7,0.011231999844312668
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,15,0.011194666226704916
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,15,0.012666666259368261
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,31,0.01128000020980835
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,31,0.01221866657336553
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,63,0.011002667248249054
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,63,0.01221866657336553
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,127,0.011157333850860596
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,127,0.013248000293970108
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,255,0.01221866657336553
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,255,0.013455999394257864
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,511,0.01239466667175293
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,511,0.013738666971524557
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,1023,0.012837332983811697
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,1023,0.014677333335081736
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,2047,0.014229333649079004
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,2047,0.016085332880417507
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,4095,0.016095999628305435
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,4095,0.01684800038735072
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,8191,0.018330667167901993
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,8191,0.01952533299724261
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,16383,0.022858666876951855
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,16383,0.024879999458789825
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,32767,0.03267733256022135
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,32767,0.03536533315976461
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,7,0.011221333096424738
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,1,0.011338666081428528
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,1,0.012106666962305704
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,3,0.01137599969903628
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,3,0.012298667182525
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,7,0.012298667182525
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,15,0.01145600030819575
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,15,0.012319999436537424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,31,0.011194666226704916
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,31,0.012560000022252401
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,63,0.011434666812419891
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,63,0.012458667159080505
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,127,0.011343999455372492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,127,0.013130666067202887
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,255,0.012576000144084295
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,255,0.01339200014869372
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,511,0.012730666746695837
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,511,0.01443733274936676
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,1023,0.013738666971524557
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,1023,0.015520000209410986
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,2047,0.015103999525308609
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,2047,0.016832000265518825
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,4095,0.016234666109085083
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,16383,0.02593066543340683
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,4095,0.017418666432301205
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,8191,0.018453333526849747
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,1,0.012719999998807907
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,8191,0.020053333292404812
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,16383,0.023813332120577495
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,32767,0.04155199974775314
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,32767,0.036746665835380554
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,1,0.01139733319481214
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,3,0.011424000064531961
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,3,0.012282667060693106
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,7,0.011525332927703857
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,7,0.012191999703645706
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,15,0.011429333438475927
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,15,0.012304000556468964
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,31,0.011381333072980246
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,31,0.012469333906968435
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,63,0.011407999942700068
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,63,0.012272000312805176
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,127,0.011621333658695221
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,127,0.013594667116800943
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,255,0.012586666891972223
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,255,0.014218666901191076
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,511,0.013104000439246496
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,511,0.014885333677132925
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,1023,0.014122666170199713
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,4095,0.01699200024207433
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,1023,0.015157333264748255
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,2047,0.01544533297419548
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,2047,0.017082666357358296
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,4095,0.017786666750907898
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,8191,0.01876266673207283
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,8191,0.020442667106787365
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,16383,0.03166933357715607
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,16383,0.02607999990383784
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,32767,0.048623998959859215
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,32767,0.03997333347797394
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,1,0.010970667004585266
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,1,0.012096000214417776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,3,0.011029332876205444
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,3,0.012186666329701742
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,7,0.010928000013033548
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,7,0.012319999436537424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,15,0.011007999380429586
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,15,0.012053333222866058
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,31,0.011429333438475927
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,31,0.012181332955757776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,63,0.011141333729028702
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,63,0.012154666086037954
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,127,0.011018666128317514
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,127,0.013365333278973898
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,255,0.012389333297808966
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,255,0.01341333364446958
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,511,0.012485332787036896
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,511,0.013807999591032663
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,1023,0.013621332744757334
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,1023,0.014842666685581207
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,2047,0.014885333677132925
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,2047,0.01605333387851715
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,4095,0.01729600007335345
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,4095,0.018730666488409042
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,8191,0.02081599955757459
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,8191,0.023285334308942158
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,16383,0.02958933264017105
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,16383,0.03324266771475474
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,32767,0.052298665046691895
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,32767,0.04693866769472758
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,1,0.01121066634853681
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,1,0.012080000092585882
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,3,0.01119999960064888
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,3,0.01210133358836174
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,7,0.011306667079528173
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,7,0.012383999923865
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,15,0.011215999722480774
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,15,0.012560000022252401
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,127,0.011440000186363855
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,31,0.01128000020980835
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,31,0.012096000214417776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,63,0.011407999942700068
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,63,0.01221866657336553
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,127,0.013338666409254074
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,255,0.012389333297808966
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,255,0.013381333400805792
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,511,0.012960000584522883
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,511,0.014394666999578476
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,1023,0.014165333161751429
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,1023,0.015568000574906668
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,8191,0.021482666333516438
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,2047,0.01587733378012975
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,2047,0.016757333030303318
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,4095,0.018383999665578205
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,4095,0.019487999379634857
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,8191,0.023760000864664715
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,16383,0.03046400099992752
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,16383,0.03402133285999298
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,32767,0.04960533479849497
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,32767,0.056554665168126426
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,1,0.011301333705584208
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,1,0.01221866657336553
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,15,0.011557333171367645
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,3,0.011424000064531961
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,3,0.012389333297808966
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,7,0.011333333949247995
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,7,0.012181332955757776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,15,0.012335999558369318
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,127,0.011525332927703857
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,31,0.011343999455372492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,31,0.012186666329701742
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,63,0.011674666156371435
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,63,0.012576000144084295
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,127,0.01349866638580958
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,255,0.012746666868527731
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,255,0.01431999976436297
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,511,0.013194666554530462
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,511,0.01498666654030482
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,1023,0.01504533365368843
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,1023,0.01598400001724561
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,2047,0.016650666793187458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,2047,0.0161920003592968
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,4095,0.017887999614079792
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,4095,0.019493332753578823
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,8191,0.022704000274340313
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,8191,0.024538666009902954
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,32767,0.05874133110046387
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,16383,0.03426666557788849
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,16383,0.03581333408753077
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,32767,0.05333333214124044
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,1,0.011391999820868174
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,1,0.012389333297808966
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,3,0.011653333902359009
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,3,0.012495999534924826
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,7,0.011514666179815928
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,7,0.012821332861979803
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,15,0.01180800050497055
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,15,0.012586666891972223
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,31,0.01184533288081487
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,31,0.012495999534924826
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,63,0.011898666620254517
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,255,0.013637332866589228
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,63,0.01250133290886879
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,127,0.011893333246310553
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,127,0.01471466695268949
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,255,0.014416000495354334
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,511,0.013424000392357508
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,511,0.015130666395028433
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,1023,0.014991999914248785
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,1023,0.016442666451136272
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,2047,0.016672000288963318
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,2047,0.017349333812793095
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,4095,0.01865600049495697
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,4095,0.02032533288002014
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,8191,0.030613332986831665
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,8191,0.024661332368850708
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,16383,0.045370668172836304
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,16383,0.03925333420435587
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,32767,0.07753600180149078
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,32767,0.05982933441797892
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,1,0.012736000120639801
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,fp8,1,0.013445333888133367
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,3,0.013002666334311167
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,fp8,3,0.013274667163689932
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,7,0.012837332983811697
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,fp8,7,0.013253333667914072
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,15,0.013125333935022354
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,fp8,15,0.013397333522637686
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,31,0.012826666235923767
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,fp8,31,0.01357866699496905
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,63,0.012703999876976013
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,fp8,63,0.013594667116800943
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,127,0.013317332913478216
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,fp8,127,0.01479999969402949
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,255,0.014592000593741735
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,fp8,255,0.01609066625436147
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,511,0.01664000004529953
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,fp8,511,0.01806933308641116
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,1023,0.02081599955757459
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,fp8,1023,0.022639999787012737
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,2047,0.03287466615438461
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,fp8,2047,0.033215999603271484
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,4095,0.05050666630268097
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,fp8,4095,0.05842133363087972
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,1,0.013077333569526672
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,fp8,1,0.014181333283583323
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,3,0.01292266696691513
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,fp8,3,0.01402666668097178
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,7,0.013104000439246496
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,fp8,7,0.014250667144854864
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,15,0.01310933381319046
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,fp8,15,0.0143306665122509
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,31,0.013210666676362356
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,fp8,31,0.0143306665122509
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,63,0.013162666310866674
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,255,0.015157333264748255
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,fp8,63,0.014218666901191076
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,127,0.013482666263977686
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,fp8,127,0.015482666591803232
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,fp8,255,0.0162773331006368
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,511,0.016810666769742966
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,fp8,511,0.019061333189407986
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,1023,0.029669334491093952
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,fp8,2047,0.03833066672086716
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,fp8,1023,0.024266667664051056
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,2047,0.044309332966804504
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,4095,0.07548800110816956
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,3,0.017194667210181553
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,fp8,4095,0.060005332032839455
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,1,0.017008000363906223
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,fp8,1,0.018016000588734944
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,fp8,3,0.017893332988023758
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,7,0.01717866708834966
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,15,0.01718933383623759
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,fp8,7,0.017925333231687546
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,fp8,15,0.018063999712467194
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,31,0.01727466657757759
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,fp8,31,0.017984000345071156
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,63,0.017173333714405697
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,fp8,63,0.018181333939234417
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,127,0.017360000560681026
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,fp8,127,0.02111999938885371
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,fp8,511,0.027610667049884796
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,255,0.020874666670958202
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,fp8,255,0.02233600119749705
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,511,0.0317493329445521
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,1023,0.04719999929269155
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,fp8,1023,0.04201066493988037
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,2047,0.07699733475844066
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,fp8,2047,0.06388266881306966
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,4095,0.13606933752695718
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,fp8,4095,0.10577600200970967
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,1,0.024085332949956257
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,fp8,1,0.025407999753952026
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,fp8,3,0.025834667185942333
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,3,0.02388266722361247
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,7,0.02407466620206833
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,fp8,7,0.025455998877684276
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,31,0.024421334266662598
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,15,0.024117333193620045
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,fp8,15,0.025205334027608235
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,fp8,31,0.02589333305756251
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,63,0.024293333292007446
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,fp8,63,0.025493333737055462
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,127,0.02437866727511088
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,fp8,127,0.03027733415365219
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,255,0.03751466671625773
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,fp8,255,0.033600000043710075
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,511,0.05012799799442291
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,1023,0.08036800225575765
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,fp8,511,0.04983466863632202
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,2047,0.13823999961217245
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,fp8,1023,0.06921066840489705
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,fp8,2047,0.11281067132949829
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,1,0.011584000041087469
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,1,0.010677333921194077
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,4095,0.255018671353658
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,3,0.011733333269755045
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,7,0.010741333166758219
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,3,0.010725333044926325
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,7,0.01184533288081487
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,fp8,4095,0.1962933341662089
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,15,0.010672000547250112
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,15,0.01209066684047381
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,31,0.010784000158309937
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,127,0.013429333766301474
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,31,0.011760000139474869
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,63,0.010970667004585266
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,63,0.01191466674208641
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,127,0.010847999403874079
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,255,0.011999999483426413
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,255,0.013242666920026144
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,511,0.013104000439246496
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,511,0.014463999619086584
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,1023,0.014762666076421738
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,1023,0.015530666957298914
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,2047,0.015893333901961643
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,2047,0.018079999834299088
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,4095,0.019914666811625164
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,4095,0.02277333289384842
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,8191,0.028837333122889202
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,8191,0.03315199911594391
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,16383,0.046997333566347756
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,16383,0.05201066533724467
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,32767,0.08320533235867818
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,32767,0.09248000383377075
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,1,0.010709332923094431
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,1,0.0120319997270902
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,3,0.011087999989589056
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,3,0.0122079998254776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,7,0.011045332998037338
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,7,0.011920000116030375
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,15,0.010703999549150467
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,15,0.012047999848922094
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,31,0.010933333386977514
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,31,0.011994666109482447
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,63,0.010992000500361124
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,63,0.012367999802033106
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,127,0.01091733326514562
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,127,0.013434667140245438
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,255,0.012378666549921036
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,255,0.014111999422311783
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,511,0.013760000467300415
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,511,0.015589332828919092
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,1023,0.015482666591803232
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,1023,0.016330666840076447
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,2047,0.01632533346613248
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,2047,0.018778666853904724
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,4095,0.020597333709398907
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,4095,0.02332799881696701
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,8191,0.029626667499542236
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,1,0.01119999960064888
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,8191,0.03377600014209747
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,16383,0.04959466556708018
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,16383,0.05320533116658529
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,32767,0.08397333820660909
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,7,0.011440000186363855
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,32767,0.09996267159779866
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,1,0.012527999778588613
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,3,0.011183999478816986
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,3,0.012223999947309494
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,7,0.012063999970753988
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,63,0.011509332805871964
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,15,0.011551999797423681
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,15,0.012159999459981918
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,31,0.011247999966144562
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,31,0.012245333443085352
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,63,0.012256000190973282
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,127,0.011306667079528173
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,127,0.015189333508412043
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,255,0.01303999995191892
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,255,0.014165333161751429
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,511,0.01441066712141037
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,511,0.015397333850463232
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,1023,0.01581866666674614
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,1023,0.016538667182127636
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,8191,0.03236799935499827
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,2047,0.01661866654952367
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,2047,0.019280000279347103
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,4095,0.021695998807748158
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,4095,0.02363733450571696
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,8191,0.033861334125200905
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,16383,0.06204266846179962
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,16383,0.05825066566467285
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,32767,0.09457066655158997
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,32767,0.10109333197275798
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,1,0.011871999750534693
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,1,0.012608000387748083
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,3,0.01157333329319954
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,3,0.013061333447694778
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,7,0.011813333878914515
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,7,0.012970666090647379
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,15,0.011989332735538483
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,15,0.012997332960367203
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,31,0.011792000383138657
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,63,0.011941333611806234
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,31,0.012698666503032049
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,63,0.012991999586423239
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,127,0.011877333124478659
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,127,0.014159999787807465
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,255,0.013141332815090815
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,255,0.014618666221698126
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,511,0.014666666587193808
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,511,0.015722667177518208
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,1023,0.015754666179418564
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,1023,0.016810666769742966
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,2047,0.017583999782800674
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,2047,0.019493332753578823
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,4095,0.029637334247430164
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,4095,0.0239680012067159
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,8191,0.045040001471837364
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,8191,0.038848000268141426
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,16383,0.0757066657145818
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,16383,0.05964800218741099
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,32767,0.1355946660041809
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,32767,0.10155733426411946
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,fp8,1,0.015498666713635126
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,1,0.01414399966597557
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,3,0.014645333091417948
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,fp8,3,0.015439999600251516
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,7,0.014352000008026758
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,fp8,7,0.01571200042963028
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,15,0.014490666488806406
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,fp8,15,0.01551466683546702
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,31,0.014192000031471252
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,fp8,31,0.015392000476519266
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,63,0.014544000228246054
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,fp8,63,0.015573333948850632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,127,0.01451733335852623
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,fp8,127,0.01695999999841054
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,255,0.016602666427691776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,fp8,255,0.01782400036851565
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,511,0.0184906671444575
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,fp8,511,0.02033599962790807
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,1023,0.030949334303538006
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,fp8,1023,0.02569599946339925
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,2047,0.04598933458328247
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,fp8,2047,0.04078399886687597
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,1,0.01788266624013583
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,fp8,1,0.019274666905403137
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,3,0.017925333231687546
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,7,0.017903999735911686
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,fp8,3,0.019141333798567455
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,fp8,7,0.019061333189407986
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,15,0.01775466650724411
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,31,0.01770666614174843
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,fp8,15,0.019434666881958645
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,fp8,31,0.019317333896954853
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,63,0.017866666118303936
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,fp8,63,0.01932799940307935
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,127,0.018186666071414948
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,fp8,127,0.021957332889238994
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,255,0.02110933264096578
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,fp8,255,0.023525332411130268
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,511,0.03373866776625315
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,fp8,511,0.029253333806991577
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,1023,0.04981866478919983
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,fp8,1023,0.044250667095184326
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,1,0.024864000578721363
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,fp8,1,0.026538667579491932
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,2047,0.07876800000667572
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,fp8,2047,0.06562133133411407
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,3,0.02442666639884313
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,fp8,3,0.026362667481104534
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,15,0.02476266771554947
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,7,0.02460266649723053
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,fp8,15,0.026421333352724712
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,fp8,7,0.026533332963784535
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,63,0.024869332710901897
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,31,0.024634666740894318
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,fp8,63,0.026357332865397137
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,127,0.025631998976071674
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,fp8,127,0.030991998811562855
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,255,0.03751466671625773
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,fp8,31,0.026496000587940216
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,fp8,255,0.034703999757766724
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,511,0.05146133402983347
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,fp8,511,0.05068266888459524
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,2047,0.1388213336467743
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,fp8,2047,0.11379733681678772
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,1023,0.0821973333756129
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,fp8,1023,0.07102933526039124
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,1,0.03756266583998998
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,fp8,1,0.04013866682847341
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,3,0.03746666759252548
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,fp8,3,0.04093866546948751
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,7,0.037632000943024956
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,fp8,15,0.0405173326532046
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,fp8,7,0.04055466751257578
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,31,0.03755733370780945
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,fp8,31,0.04036800066630045
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,15,0.03788266579310099
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,63,0.03834133346875509
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,fp8,63,0.04032533367474874
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,127,0.044794668753941856
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,fp8,127,0.049498667319615684
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,255,0.058602665861447654
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,fp8,255,0.061109334230422974
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,1023,0.14506666858990988
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,fp8,1023,0.12408533692359924
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,2047,0.25933865706125897
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,fp8,2047,0.21076800425847372
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,1,0.0207893339296182
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,fp8,511,0.0846560001373291
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,511,0.08895466725031535
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,fp8,1,0.022554665803909302
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,3,0.0206986665725708
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,fp8,3,0.02223466585079829
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,7,0.020560000091791153
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,fp8,7,0.02222399910291036
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,15,0.020768000433842342
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,fp8,31,0.02239999920129776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,fp8,63,0.02253866692384084
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,63,0.020981334149837494
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,fp8,127,0.02499199906984965
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,127,0.021104000508785248
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,255,0.024608001112937927
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,1,0.026736001173655193
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,fp8,255,0.0268053337931633
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,fp8,1,0.029157333076000214
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,3,0.0271573339899381
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,511,0.03728533287843069
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,31,0.02057066683967908
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,fp8,15,0.02250666668017705
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,fp8,3,0.028954667349656422
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,fp8,511,0.032842665910720825
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,7,0.02699200063943863
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,fp8,7,0.029370665550231934
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,15,0.027061333258946735
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,fp8,15,0.02917333443959554
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,31,0.02716800073782603
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,fp8,31,0.029557332396507263
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,63,0.027029333015282948
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,fp8,63,0.029306667546431225
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,127,0.028016000986099243
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,fp8,127,0.03342399994532267
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,255,0.04178666571776072
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,fp8,255,0.038149334490299225
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,1,0.0399893323580424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,fp8,511,0.054048001766204834
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,511,0.057018667459487915
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,fp8,1,0.04308799902598063
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,3,0.040063999593257904
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,fp8,3,0.04292266567548116
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,7,0.04011200120051702
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,fp8,7,0.043050666650136314
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,15,0.04002666721741358
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,fp8,15,0.04367466767628988
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,fp8,31,0.04299733539422353
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,63,0.041333332657814026
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,fp8,63,0.04313066601753235
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,31,0.04045333216587702
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,127,0.04650133351484934
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,fp8,127,0.05308799942334493
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,255,0.06081066528956095
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,fp8,255,0.0655626654624939
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,1,0.06612800061702728
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,fp8,1,0.07030933101971944
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,511,0.09211200475692749
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,fp8,511,0.08747733632723491
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,3,0.06618133187294006
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,fp8,3,0.07062399884064992
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,7,0.06614933411280315
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,15,0.06654400130112965
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,fp8,7,0.07099733253320058
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,fp8,15,0.07046933472156525
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,63,0.0721013347307841
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,fp8,63,0.07306133210659027
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,127,0.07443733513355255
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,31,0.06749333441257477
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,fp8,31,0.0702453354994456
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,fp8,127,0.09353066484133403
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,255,0.10175466537475586
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,fp8,255,0.10845866799354553
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,1,0.037434667348861694
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,511,0.16246933738390604
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,fp8,511,0.15460800131162009
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,fp8,1,0.03914133210976919
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,3,0.03782399992148081
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,fp8,7,0.03935466706752777
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,fp8,3,0.03935466706752777
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,31,0.0378506655494372
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,7,0.03762666632731756
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,fp8,31,0.03959999978542328
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,63,0.03791466603676478
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,fp8,63,0.0395359992980957
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,fp8,15,0.03913066784540812
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,fp8,127,0.04311466713746389
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,127,0.039135999977588654
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,15,0.0377866675456365
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,255,0.0525546669960022
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,fp8,255,0.048954665660858154
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,1,0.049456000328063965
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,fp8,1,0.052058666944503784
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,3,0.04916266600290934
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,fp8,3,0.05172266562779745
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,7,0.04929600159327189
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,fp8,7,0.051967998345692955
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,15,0.04964800179004669
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,fp8,15,0.05262400209903717
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,31,0.049546668926874794
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,fp8,31,0.05194133520126343
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,63,0.05124799907207489
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,fp8,63,0.051957334081331887
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,127,0.058517331878344216
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,fp8,127,0.06345599889755249
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,255,0.07400000095367432
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,fp8,255,0.07686399916807811
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,1,0.07549866537253062
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,fp8,1,0.08109333117802937
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,3,0.07569066683451335
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,fp8,3,0.08090133468310039
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,7,0.07566933333873749
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,fp8,7,0.08116800089677174
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,15,0.07640000184377034
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,31,0.07893866797288258
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,fp8,15,0.08181866506735484
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,fp8,31,0.08134933312733968
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,63,0.08290666838486989
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,fp8,63,0.08246933420499165
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,fp8,127,0.10365333159764607
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,127,0.0865119993686676
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,255,0.11204800009727478
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,1,0.12851199507713318
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,fp8,1,0.1344533363978068
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,fp8,255,0.12054399649302165
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,3,0.12795733412106833
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,fp8,3,0.1345919966697693
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,7,0.12840533256530762
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,fp8,7,0.13453867038091025
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,fp8,15,0.13660800457000732
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,31,0.1349440018335978
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,15,0.13114666938781738
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,63,0.1362986663977305
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,fp8,63,0.14131733775138855
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,1,0.011136000355084738
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,3,0.01210133358836174
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,fp8,31,0.13750400145848593
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,1,0.011978667229413986
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,3,0.011120000233252844
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,7,0.011050666371981302
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,7,0.01201066623131434
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,15,0.01102399950226148
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,15,0.0120319997270902
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,fp8,127,0.17270400126775107
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,31,0.011194666226704916
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,127,0.14267733693122864
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,31,0.012133333832025528
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,63,0.011055999745925268
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,63,0.012047999848922094
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,127,0.013370666652917862
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,127,0.011168000598748526
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,255,0.014287999520699183
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,511,0.013616000612576803
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,255,0.012335999558369318
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,511,0.01526933287580808
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,1023,0.015813333292802174
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,fp8,255,0.20298133293787637
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,1023,0.018005333840847015
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,255,0.19031467040379843
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,2047,0.020101333657900494
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,2047,0.022618666291236877
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,4095,0.02888533224662145
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,4095,0.03262399882078171
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,8191,0.046725332736968994
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,16383,0.08295999964078267
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,32767,0.15219199657440186
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,32767,0.18440000216166177
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,1,0.011343999455372492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,1,0.012213333199421564
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,8191,0.052058666944503784
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,3,0.011365332951148352
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,3,0.01209066684047381
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,16383,0.09708266456921895
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,7,0.011136000355084738
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,7,0.01201066623131434
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,15,0.011370666325092316
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,15,0.012319999436537424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,63,0.01190399999419848
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,31,0.01137599969903628
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,31,0.012144000579913458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,63,0.011594666788975397
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,127,0.011301333705584208
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,127,0.013295999417702356
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,255,0.012709333250919977
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,255,0.015802666544914246
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,511,0.014474666366974512
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,511,0.016490666816631954
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,1023,0.01659199967980385
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,1023,0.018325333793958027
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,2047,0.020714666694402695
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,2047,0.023418667415777843
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,8191,0.05399466554323832
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,16383,0.0844640036424001
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,4095,0.029722665747006733
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,4095,0.03333866596221924
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,8191,0.049685334165891014
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,1,0.011450666934251785
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,16383,0.1009333332379659
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,32767,0.15480533242225647
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,7,0.011600000162919363
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,32767,0.1851573387781779
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,1,0.01303999995191892
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,3,0.011823999385039011
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,3,0.012810666114091873
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,7,0.012986666212479273
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,15,0.011733333269755045
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,15,0.012746666868527731
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,31,0.011866666376590729
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,31,0.013050666699806849
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,63,0.011813333878914515
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,255,0.01563199982047081
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,63,0.012954667210578918
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,127,0.011717333147923151
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,127,0.014650666465361914
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,255,0.013237333546082178
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,511,0.014661333213249842
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,511,0.016634666671355564
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,1023,0.016362667083740234
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,1023,0.019120000302791595
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,2047,0.021509334444999695
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,2047,0.023786666492621105
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,4095,0.038032000263532005
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,4095,0.034474665919939675
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,8191,0.05508799850940704
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,8191,0.058362667759259544
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,16383,0.11169600486755371
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,16383,0.10087466239929199
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,32767,0.17249067624409994
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,32767,0.1832586725552877
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,1,0.011978667229413986
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,1,0.013082666943470636
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,3,0.01198400060335795
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,3,0.013295999417702356
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,7,0.011898666620254517
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,7,0.012831999609867731
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,15,0.012042666474978128
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,15,0.012885333349307379
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,31,0.012074666718641916
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,31,0.012975999464591345
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,63,0.01181866725285848
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,63,0.013466666142145792
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,127,0.012240000069141388
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,127,0.01443733274936676
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,255,0.013290667285521826
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,255,0.016165333489576977
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,511,0.015285332997639975
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,511,0.01739199956258138
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,1023,0.017136000096797943
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,1023,0.019317333896954853
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,2047,0.029648000995318096
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,2047,0.02534399926662445
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,4095,0.04570133487383524
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,4095,0.03854399919509888
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,8191,0.0765066643555959
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,8191,0.060362666845321655
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,16383,0.13586666186650595
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,16383,0.10220799843470256
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,32767,0.25860800345738727
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,32767,0.1859040061632792
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,1,0.038405333956082664
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,fp8,1,0.038922667503356934
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,3,0.03874133278926214
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,fp8,3,0.03920000046491623
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,7,0.03826666623353958
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,fp8,7,0.03937066594759623
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,15,0.038176000118255615
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,fp8,15,0.03926933308442434
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,31,0.03869866579771042
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,fp8,31,0.039274667700131737
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,63,0.04253333310286204
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,1,0.06570133566856384
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,fp8,63,0.039264000952243805
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,127,0.04659733176231384
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,fp8,127,0.050330668687820435
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,fp8,1,0.06340266764163971
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,3,0.06435733536879222
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,fp8,3,0.06306133170922597
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,7,0.06484800080458324
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,fp8,7,0.06279466549555461
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,fp8,15,0.0631573349237442
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,15,0.06570666531721751
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,31,0.06955733398596446
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,fp8,31,0.06345599889755249
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,63,0.07196799914042155
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,fp8,63,0.06800533334414165
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,127,0.0746506651242574
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,1,0.11601600050926208
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,fp8,127,0.08803199728329976
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,fp8,1,0.11462400356928508
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,3,0.11575999855995178
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,fp8,3,0.113946666320165
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,7,0.11582932869593303
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,fp8,7,0.11387733618418376
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,fp8,15,0.11434132854143779
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,15,0.12367467085520427
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,31,0.12702932953834534
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,fp8,31,0.12125866611798604
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,63,0.12869866689046225
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,fp8,63,0.12544000148773193
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,127,0.13245866696039835
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,fp8,127,0.15874133507410684
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,1,0.21224000056584677
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,fp8,1,0.20674665768941244
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,3,0.2121866742769877
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,fp8,3,0.20888533194859824
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,7,0.22540799776713052
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,fp8,7,0.20785599946975708
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,31,0.230186661084493
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,15,0.22839999198913574
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,fp8,15,0.21772799889246622
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,fp8,31,0.2227840026219686
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,63,0.23174933592478433
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,fp8,63,0.2247520089149475
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,127,0.24067733685175577
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,fp8,127,0.2927839954694112
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,1,0.06541333099206288
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,fp8,1,0.06630399823188782
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,3,0.06635199983914693
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,fp8,3,0.06635733445485432
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,7,0.06613866488138835
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,fp8,7,0.06654400130112965
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,15,0.0708426684141159
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,fp8,15,0.06695466736952464
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,31,0.07372800012429555
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,fp8,31,0.06900266806284587
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,63,0.07478933533032735
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,fp8,63,0.07618133227030437
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,1,0.11661866307258606
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,fp8,1,0.11463466286659241
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,3,0.1183733344078064
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,fp8,3,0.11398933331171672
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,7,0.12346667051315308
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,fp8,7,0.11334400375684102
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,15,0.12894399960835776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,fp8,15,0.1160586675008138
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,31,0.12897066275278726
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,fp8,31,0.12564266721407572
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,63,0.13114133477210999
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,fp8,63,0.12756266196568808
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,1,0.2227893273035685
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,fp8,1,0.21422932545344034
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,3,0.23145600159962973
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,fp8,3,0.21494932969411215
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,7,0.2376533349355062
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,fp8,7,0.21998933951059976
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,15,0.2403200070063273
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,fp8,15,0.2313013275464376
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,31,0.24045334259668985
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,fp8,31,0.23212265968322754
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,63,0.2433919906616211
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,1,0.42981866995493573
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,fp8,63,0.23492266734441122
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,fp8,1,0.4052480061848958
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,3,0.4439306656519572
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,fp8,3,0.4124266703923543
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,7,0.4455466667811076
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,1,0.01119999960064888
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,1,0.012037333101034164
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,3,0.01129066695769628
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,3,0.012335999558369318
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,fp8,7,0.4226826826731364
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,7,0.011472000430027643
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,15,0.44629331429799396
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,15,0.011296000331640244
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,31,0.44679466883341473
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,7,0.012341332932313284
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,fp8,15,0.4280906518300374
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,31,0.011370666325092316
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,15,0.012245333443085352
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,63,0.012245333443085352
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,127,0.01146666705608368
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,127,0.013546666751305262
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,63,0.01146666705608368
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,31,0.012458667159080505
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,255,0.012719999998807907
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,255,0.014127999544143677
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,511,0.014815999815861383
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,511,0.017114666601022083
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,1023,0.01903466631968816
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,1023,0.021562665700912476
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,2047,0.027802666028340656
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,2047,0.031856000423431396
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,4095,0.04636266827583313
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,4095,0.05067733426888784
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,8191,0.08189333478609721
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,fp8,31,0.4310186703999837
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,8191,0.09315199653307597
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,16383,0.15213867028554282
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,1,0.011887999872366587
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,16383,0.1821333368619283
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,1,0.01333333303531011
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,3,0.012970666090647379
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,7,0.011829332758982977
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,7,0.012810666114091873
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,3,0.012096000214417776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,15,0.01190399999419848
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,15,0.012954667210578918
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,31,0.011941333611806234
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,31,0.012842666357755661
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,63,0.011952000359694162
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,63,0.013072000195582708
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,127,0.011855999628702799
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,127,0.014165333161751429
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,255,0.013248000293970108
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,255,0.01526933287580808
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,511,0.015397333850463232
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,fp8,63,0.4329386552174886
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,511,0.01766933376590411
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,1023,0.019866666446129482
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,63,0.4519733190536499
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,1023,0.0225600004196167
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,2047,0.028789333999156952
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,2047,0.031983998914559685
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,4095,0.048021331429481506
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,4095,0.05221866567929586
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,8191,0.099263995885849
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,16383,0.1545866628487905
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,3,0.012229333321253458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,1,0.011994666109482447
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,1,0.013210666676362356
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,3,0.0129120002190272
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,8191,0.08334933718045552
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,7,0.011749333391586939
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,7,0.013248000293970108
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,15,0.012266666938861212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,63,0.012186666329701742
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,15,0.013114667187134424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,16383,0.18266133467356363
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,31,0.012202666451533636
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,31,0.01302933320403099
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,63,0.013274667163689932
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,127,0.01240533341964086
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,127,0.015194666882356008
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,255,0.013818666338920593
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,255,0.015344000111023584
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,511,0.01569066693385442
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,511,0.017973333597183228
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,4095,0.051781331499417625
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,1023,0.020314666132132213
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,1023,0.022448000808556873
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,2047,0.03230399886767069
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,2047,0.03323733309904734
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,4095,0.05705599983533224
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,8191,0.09217600027720134
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,8191,0.09902933239936829
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,16383,0.1567253371079763
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,16383,0.18324265877405801
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,1,0.01231466606259346
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,1,0.013503999759753546
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,3,0.012362666428089142
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,3,0.013493333011865616
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,7,0.01209066684047381
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,7,0.01332266628742218
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,15,0.012245333443085352
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,15,0.013440000514189402
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,31,0.012272000312805176
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,31,0.013408000270525614
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,63,0.012565333396196365
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,63,0.013637332866589228
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,127,0.012613333761692047
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,127,0.015002666662136713
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,255,0.014159999787807465
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,1023,0.02310933421055476
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,255,0.01584533353646596
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,2047,0.03832533210515976
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,511,0.01637866720557213
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,511,0.018437333405017853
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,1023,0.02842666705449422
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,2047,0.0429066667954127
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,4095,0.0738560010989507
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,4095,0.0587360014518102
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,8191,0.13449066877365112
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,8191,0.10106666882832845
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,16383,0.18600000937779745
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,16383,0.25642667214075726
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,1,0.0120319997270902
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,1,0.013050666699806849
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,7,0.013359999905029932
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,3,0.011717333147923151
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,3,0.01267733300725619
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,7,0.011711999773979187
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,15,0.011839999506870905
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,15,0.012634667257467905
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,31,0.012117333710193634
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,31,0.012949333836634954
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,63,0.011920000116030375
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,63,0.012703999876976013
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,127,0.0120319997270902
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,127,0.014474666366974512
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,255,0.01328533391157786
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,1023,0.019466667125622433
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,255,0.015354666858911514
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,511,0.015568000574906668
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,511,0.017418666432301205
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,1023,0.02293866624434789
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,2047,0.028783999383449554
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,2047,0.03216533362865448
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,4095,0.048170665899912514
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,4095,0.052629331747690834
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,8191,0.08436266581217448
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,8191,0.1002293328444163
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,16383,0.15381866693496704
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,16383,0.1851146618525187
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,1,0.012362666428089142
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,1,0.012794667234023413
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,3,0.012144000579913458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,3,0.01392000044385592
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,7,0.012133333832025528
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,7,0.013173333058754602
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,15,0.012149333953857422
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,15,0.013045333325862885
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,31,0.0120319997270902
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,63,0.012469333906968435
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,31,0.013114667187134424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,63,0.013461332768201828
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,127,0.012554666648308435
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,127,0.014261333892742792
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,255,0.014442666123310724
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,255,0.015024000157912573
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,511,0.015504000087579092
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,511,0.017504000415404636
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,1023,0.020090666910012562
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,1023,0.022757334013779957
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,2047,0.03102933367093404
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,2047,0.03267733256022135
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,4095,0.049440001447995506
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,4095,0.054458667834599815
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,8191,0.09410666426022847
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,8191,0.09431999921798706
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,16383,0.17356799046198526
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,3,0.013631999492645264
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,16383,0.17443732420603433
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,1,0.012223999947309494
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,1,0.013573333621025085
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,3,0.012266666938861212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,7,0.012351999680201212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,7,0.013546666751305262
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,63,0.01259200026591619
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,15,0.013408000270525614
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,15,0.01251199965675672
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,31,0.0124746672809124
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,31,0.013487999637921652
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,63,0.013642666240533194
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,127,0.012597333639860153
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,127,0.01526933287580808
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,255,0.014757333944241205
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,255,0.01591466615597407
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,511,0.016063999384641647
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,511,0.018629333625237148
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,1023,0.02777066578467687
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,1023,0.023423999547958374
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,2047,0.04337066908677419
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,2047,0.03793066740036011
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,4095,0.0744053324063619
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,4095,0.05853866537412008
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,16383,0.18627200524012247
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,8191,0.13487999637921652
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,8191,0.10186666250228882
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,16383,0.25390400489171344
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,1,0.016458666572968166
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,1,0.01754666616519292
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,3,0.016442666451136272
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,3,0.017605333278576534
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,7,0.016421332955360413
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,7,0.017583999782800674
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,15,0.016506666938463848
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,15,0.0176959993938605
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,31,0.016607999801635742
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,31,0.017717332889636356
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,63,0.016336000214020412
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,63,0.017818666994571686
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,127,0.01717866708834966
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,127,0.020794666061798733
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,255,0.01989866668979327
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,511,0.02754133443037669
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,255,0.021935999393463135
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,511,0.030762667457262676
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,1023,0.046341334780057274
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,1023,0.04138133426507314
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,2047,0.07543999950091045
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,2047,0.06292800108591716
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,4095,0.1357493301232656
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,4095,0.10469333330790202
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,8191,0.2534986734390259
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,8191,0.198527991771698
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,1,0.010981333752473196
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,1,0.012362666428089142
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,16383,0.49242134888966876
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,3,0.011050666371981302
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,16383,0.3569973309834798
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,3,0.012154666086037954
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,7,0.010992000500361124
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,7,0.0120319997270902
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,15,0.011125333607196808
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,15,0.012106666962305704
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,31,0.01091733326514562
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,31,0.012058666596810022
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,63,0.011242666592200598
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,63,0.012223999947309494
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,127,0.011045332998037338
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,127,0.013408000270525614
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,255,0.012319999436537424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,255,0.013365333278973898
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,511,0.012362666428089142
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,511,0.013770667215188345
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,1023,0.012682666381200155
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,1023,0.014058666924635569
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,2047,0.013904000322024027
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,2047,0.015274666249752045
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,4095,0.015290666371583939
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,4095,0.01642666632930438
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,8191,0.017551999539136887
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,8191,0.019178666174411774
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,16383,0.02229333420594533
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,16383,0.02476799984773
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,32767,0.03181333343187968
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,32767,0.03481066723664602
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,65535,0.05009600023428599
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,65535,0.05551466842492422
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,1,0.011183999478816986
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,1,0.011866666376590729
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,3,0.011157333850860596
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,3,0.012138667205969492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,7,0.01110400011142095
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,7,0.012229333321253458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,15,0.01129066695769628
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,15,0.01210133358836174
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,31,0.011359999577204386
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,31,0.012213333199421564
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,63,0.011274666835864386
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,63,0.01219733307758967
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,127,0.011215999722480774
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,127,0.01339200014869372
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,255,0.0122079998254776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,255,0.01322666679819425
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,2047,0.014127999544143677
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,511,0.012618667135636011
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,4095,0.016021333634853363
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,4095,0.01730666682124138
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,511,0.013637332866589228
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,1023,0.012949333836634954
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,1023,0.014293332894643148
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,2047,0.015439999600251516
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,8191,0.01851733277241389
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,8191,0.019146667172511418
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,16383,0.022682666778564453
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,16383,0.02476266771554947
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,32767,0.03379199902216593
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,32767,0.034789333740870156
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,65535,0.05299733579158783
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,65535,0.057855998476346336
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,1,0.011114666859308878
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,1,0.012165332833925882
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,3,0.011253333340088526
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,3,0.012234666695197424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,7,0.011519999553759893
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,7,0.01219733307758967
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,15,0.011328000575304031
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,15,0.012437333663304647
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,31,0.011221333096424738
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,31,0.012144000579913458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,63,0.011098666737476984
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,63,0.012416000167528788
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,127,0.011482667177915573
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,127,0.01341333364446958
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,255,0.012469333906968435
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,255,0.01350933313369751
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,511,0.012634667257467905
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,511,0.014335999886194864
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,1023,0.013722666849692663
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,1023,0.015482666591803232
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,2047,0.014912000546852747
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,2047,0.01634666696190834
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,4095,0.016330666840076447
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,4095,0.017450666675964992
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,8191,0.0185759998857975
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,8191,0.0199946661790212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,16383,0.023247999449570973
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,65535,0.05737066765626272
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,16383,0.02566933383544286
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,32767,0.035589332381884255
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,32767,0.036837334434191384
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,65535,0.061568001906077065
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,1,0.011461333682139715
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,1,0.012485332787036896
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,3,0.011328000575304031
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,3,0.012266666938861212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,7,0.011434666812419891
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,7,0.012362666428089142
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,15,0.011418666690587997
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,15,0.012282667060693106
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,127,0.011546666423479715
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,31,0.011648000528415045
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,31,0.012133333832025528
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,63,0.011695999652147293
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,63,0.01227733368674914
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,127,0.013397333522637686
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,255,0.01258133351802826
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,255,0.01414399966597557
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,511,0.013327999661366144
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,2047,0.015493333339691162
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,511,0.014511999984582266
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,1023,0.013471999516089758
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,1023,0.015253332753976187
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,2047,0.016506666938463848
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,4095,0.016949333250522614
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,4095,0.017968000223239262
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,8191,0.018976000448067982
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,8191,0.020442667106787365
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,16383,0.030938667555650074
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,16383,0.02717866748571396
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,32767,0.046853333711624146
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,32767,0.0402453343073527
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,65535,0.07932266592979431
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,65535,0.0632533331712087
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,1,0.011194666226704916
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,7,0.012256000190973282
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,1,0.012053333222866058
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,3,0.011077333241701126
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,3,0.012026666353146235
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,7,0.011045332998037338
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,15,0.011231999844312668
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,15,0.012522666404644648
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,31,0.011125333607196808
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,31,0.012149333953857422
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,63,0.010901333143313726
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,63,0.012213333199421564
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,127,0.01098666712641716
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,127,0.01333333303531011
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,255,0.01240533341964086
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,255,0.013733333597580591
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,511,0.012586666891972223
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,511,0.01358933374285698
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,1023,0.013584000368913015
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,1023,0.014581333845853806
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,2047,0.014789332946141561
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,2047,0.016143999993801117
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,4095,0.017375999440749485
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,4095,0.018677332748969395
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,8191,0.02045866722861926
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,32767,0.05231466889381409
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,65535,0.0835040012995402
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,8191,0.023311999936898548
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,16383,0.029296000798543293
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,16383,0.033376000821590424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,32767,0.04744533201058706
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,65535,0.09687999884287517
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,1,0.011306667079528173
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,1,0.012154666086037954
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,3,0.011274666835864386
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,3,0.012256000190973282
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,7,0.011317333827416102
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,7,0.012357333054145178
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,15,0.011301333705584208
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,15,0.012586666891972223
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,31,0.011391999820868174
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,31,0.012128000458081564
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,63,0.011642667154471079
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,63,0.012074666718641916
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,127,0.011359999577204386
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,127,0.01314666618903478
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,255,0.012266666938861212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,2047,0.016058667252461117
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,255,0.013568000247081121
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,511,0.012624000509579977
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,511,0.014053333550691605
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,1023,0.013818666338920593
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,1023,0.015482666591803232
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,2047,0.01617066686352094
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,4095,0.01786133274435997
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,4095,0.019717333217461903
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,8191,0.02144533395767212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,8191,0.023557332654794056
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,16383,0.030752000709374745
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,16383,0.03376533339420954
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,32767,0.04863999783992767
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,32767,0.0544053316116333
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,65535,0.08422399560610454
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,65535,0.09553066889444987
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,1,0.01126933346192042
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,1,0.012234666695197424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,15,0.01146666705608368
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,3,0.01138666644692421
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,3,0.012293333808581034
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,31,0.011567999919255575
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,7,0.011354666203260422
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,63,0.011296000331640244
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,7,0.012661332885424295
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,15,0.012426666915416718
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,31,0.012426666915416718
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,63,0.012442667037248611
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,127,0.011610666910807291
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,127,0.013482666263977686
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,255,0.01250133290886879
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,255,0.014581333845853806
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,511,0.013605333864688873
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,511,0.015141333142916361
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,1023,0.014752000570297241
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,1023,0.015935999651749928
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,2047,0.016127999871969223
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,4095,0.018197332819302876
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,2047,0.017194667210181553
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,4095,0.02011200040578842
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,8191,0.02237333357334137
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,8191,0.02474133421977361
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,16383,0.03456533451875051
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,16383,0.035546667873859406
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,32767,0.05332266787687937
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,32767,0.05841066439946493
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,65535,0.09328533212343852
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,65535,0.10132267077763875
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,1,0.011450666934251785
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,1,0.012586666891972223
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,3,0.011584000041087469
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,3,0.012479999413092932
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,7,0.011648000528415045
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,7,0.01240533341964086
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,15,0.011472000430027643
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,15,0.012565333396196365
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,31,0.011429333438475927
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,31,0.012549333274364471
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,63,0.011600000162919363
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,63,0.01259200026591619
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,127,0.01166933278242747
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,127,0.014042666802803675
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,255,0.013552000125249227
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,255,0.014869333555301031
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,511,0.013760000467300415
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,511,0.015194666882356008
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,1023,0.01492799942692121
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,1023,0.01648533344268799
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,2047,0.016506666938463848
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,2047,0.017338667064905167
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,4095,0.018895999838908512
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,4095,0.02038399999340375
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,8191,0.03019733230272929
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,8191,0.024906667570273083
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,16383,0.04573333263397217
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,16383,0.039349332451820374
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,32767,0.07610133290290833
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,32767,0.05973333120346069
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,65535,0.1357599993546804
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,3,0.012703999876976013
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,65535,0.1030453344186147
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,1,0.012810666114091873
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,fp8,1,0.013269333789745966
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,fp8,3,0.013466666142145792
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,7,0.013034666577974955
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,fp8,7,0.013616000612576803
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,15,0.012853333105643591
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,fp8,15,0.01331199953953425
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,31,0.013093333691358566
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,fp8,31,0.013343999783198038
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,63,0.012794667234023413
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,fp8,63,0.013738666971524557
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,127,0.01293333371480306
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,fp8,127,0.01470400020480156
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,255,0.014650666465361914
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,fp8,255,0.015840000162522
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,511,0.016165333489576977
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,fp8,511,0.018357332795858383
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,1023,0.020495999604463577
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,fp8,1023,0.023269332945346832
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,2047,0.03791466603676478
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,fp8,2047,0.032586666444937386
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,4095,0.05410666763782501
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,fp8,4095,0.05761066575845083
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,8191,0.08563733100891113
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,fp8,8191,0.10089066624641418
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,1,0.013178666432698568
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,fp8,1,0.01402666668097178
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,3,0.013306666165590286
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,15,0.013471999516089758
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,fp8,7,0.013855999956528345
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,fp8,3,0.013834666460752487
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,7,0.013189333180586496
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,fp8,15,0.013760000467300415
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,31,0.013258667041858038
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,fp8,31,0.013807999591032663
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,63,0.01349866638580958
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,fp8,127,0.015295999745527903
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,fp8,63,0.01414399966597557
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,127,0.013616000612576803
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,255,0.015087999403476715
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,fp8,255,0.01603200038274129
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,511,0.017407999684413273
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,fp8,511,0.01889066646496455
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,1023,0.02916266769170761
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,fp8,1023,0.023946667710940044
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,fp8,4095,0.05700799822807312
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,8191,0.1365173359711965
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,2047,0.04510400195916494
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,fp8,2047,0.03755199909210205
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,fp8,1,0.018325333793958027
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,4095,0.07547200222810109
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,fp8,3,0.018191999445358913
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,7,0.017045332739750545
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,fp8,8191,0.09803199768066406
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,1,0.017279999951521557
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,3,0.017077332983414333
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,63,0.0173333336909612
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,15,0.01730666682124138
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,127,0.017504000415404636
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,fp8,7,0.01820266619324684
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,31,0.01695999999841054
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,fp8,15,0.017925333231687546
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,fp8,31,0.018053332964579265
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,fp8,63,0.01828266680240631
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,fp8,127,0.021151999632517498
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,255,0.02083733429511388
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,fp8,255,0.022517333428064983
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,511,0.03215999901294708
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,fp8,511,0.027632000545660656
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,1023,0.04683733483155569
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,2047,0.0765173335870107
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,fp8,1023,0.04215466479460398
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,fp8,2047,0.06357333560784657
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,4095,0.1362613340218862
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,fp8,4095,0.10544533530871074
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,8191,0.2521386742591858
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,fp8,1,0.02566933383544286
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,fp8,8191,0.19065600633621216
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,1,0.02407466620206833
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,3,0.02404800057411194
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,fp8,3,0.025429333249727886
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,fp8,15,0.02589333305756251
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,7,0.02421333392461141
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,fp8,7,0.025370667378107708
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,15,0.024170666933059692
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,31,0.024031999210516613
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,fp8,31,0.025360000630219776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,63,0.024501333634058636
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,fp8,63,0.025386666258176167
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,255,0.03711466739575068
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,127,0.02497600018978119
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,fp8,127,0.03030933439731598
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,fp8,255,0.033530667424201965
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,511,0.051072001457214355
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,2047,0.1381653348604838
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,fp8,511,0.04996266464392344
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,1023,0.08036266764005025
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,fp8,1023,0.06888000170389812
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,fp8,2047,0.11329600214958191
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,4095,0.2553386688232422
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,1,0.011690666278203329
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,3,0.010602666685978571
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,fp8,4095,0.19434134165445963
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,3,0.012309333930412928
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,fp8,8191,0.361189325650533
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,1,0.010656000425418219
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,7,0.010821333775917688
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,7,0.011653333902359009
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,8191,0.4888639847437541
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,15,0.01071999967098236
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,15,0.011765333513418833
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,31,0.011125333607196808
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,31,0.011786667009194693
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,63,0.010565333068370819
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,63,0.011584000041087469
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,127,0.010778666784365972
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,127,0.012879999975363413
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,255,0.012106666962305704
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,255,0.013386666774749756
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,511,0.013349333157142004
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,511,0.014229333649079004
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,1023,0.0145066666106383
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,1023,0.015360000232855478
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,2047,0.015717333803574245
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,2047,0.018383999665578205
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,4095,0.020143999407688778
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,4095,0.022863999009132385
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,8191,0.02846933404604594
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,8191,0.032773333291212715
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,16383,0.04693333307902018
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,16383,0.05209066470464071
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,32767,0.08310399949550629
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,32767,0.09144000212351482
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,65535,0.1527466674645742
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,3,0.011136000355084738
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,65535,0.18361600240071616
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,1,0.010960000256697336
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,1,0.011877333124478659
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,3,0.01166933278242747
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,7,0.010853332777818045
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,7,0.012042666474978128
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,15,0.011328000575304031
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,15,0.011871999750534693
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,31,0.01110400011142095
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,31,0.011637333780527115
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,63,0.011018666128317514
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,63,0.011877333124478659
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,127,0.011120000233252844
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,127,0.013045333325862885
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,1023,0.015189333508412043
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,255,0.012335999558369318
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,255,0.013653332988421122
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,4095,0.02053333322207133
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,511,0.01349866638580958
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,511,0.015119999647140503
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,1023,0.01579733317097028
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,2047,0.01700266698996226
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,2047,0.01826133330663045
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,16383,0.052517334620157875
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,4095,0.022954667607943218
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,8191,0.029535998900731403
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,8191,0.03313066562016805
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,16383,0.04878933231035868
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,32767,0.08357333143552144
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,32767,0.09547733267148335
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,65535,0.15359466274579367
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,65535,0.17214399576187134
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,1,0.011120000233252844
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,1,0.01231466606259346
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,3,0.01118933285276095
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,3,0.012554666648308435
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,7,0.01139733319481214
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,7,0.012362666428089142
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,15,0.011391999820868174
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,15,0.012298667182525
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,31,0.011285333583752314
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,31,0.012319999436537424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,63,0.011317333827416102
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,63,0.012213333199421564
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,127,0.011391999820868174
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,127,0.014106666048367819
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,255,0.012736000120639801
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,255,0.01413333291808764
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,511,0.015557333827018738
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,511,0.014426667243242264
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,4095,0.021503999829292297
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,1023,0.015141333142916361
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,1023,0.01674666628241539
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,2047,0.016741332908471424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,2047,0.019253333409627277
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,4095,0.023455999791622162
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,8191,0.0378560001651446
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,8191,0.03563733398914337
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,16383,0.05454933146635691
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,65535,0.18356800079345703
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,1,0.01173866664369901
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,16383,0.05825600028038025
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,32767,0.11060800155003865
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,32767,0.10016533732414246
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,65535,0.15729600191116333
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,1,0.012842666357755661
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,3,0.011514666179815928
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,3,0.012901333471139273
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,7,0.011616000284751257
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,7,0.012757333616415659
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,63,0.011472000430027643
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,15,0.012853333105643591
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,15,0.011600000162919363
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,31,0.011781333635250727
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,31,0.012682666381200155
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,63,0.013023999830087027
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,127,0.011946666985750198
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,127,0.014122666170199713
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,255,0.013077333569526672
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,255,0.014458666245142618
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,511,0.016197333733240765
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,511,0.014570667097965876
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,1023,0.01598400001724561
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,1023,0.016688000410795212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,2047,0.01736533393462499
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,2047,0.019306667149066925
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,4095,0.029669334491093952
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,4095,0.024405332903067272
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,8191,0.044581333796183266
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,8191,0.03905600061019262
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,16383,0.07604266703128815
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,16383,0.06215466558933258
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,32767,0.13546666502952576
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,32767,0.10462400317192078
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,65535,0.2582186659177144
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,65535,0.1973386605580648
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,1,0.014229333649079004
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,fp8,1,0.01543466622630755
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,3,0.014117332796255747
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,fp8,3,0.015263999501864115
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,7,0.014240000396966934
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,fp8,7,0.015216000378131866
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,fp8,15,0.015344000111023584
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,15,0.014426667243242264
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,31,0.014138666292031607
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,fp8,31,0.015477333217859268
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,63,0.014117332796255747
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,fp8,63,0.01544533297419548
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,127,0.014522666732470194
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,fp8,127,0.017024000485738117
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,255,0.016021333634853363
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,fp8,255,0.01735466718673706
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,511,0.018378666291634243
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,fp8,511,0.020282667130231857
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,1023,0.03025600065787633
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,fp8,1023,0.02518400053183238
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,2047,0.045909335215886436
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,fp8,2047,0.040607998768488564
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,1,0.017514667163292568
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,fp8,1,0.018751999984184902
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,4095,0.07605333129564922
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,fp8,4095,0.061887999375661217
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,fp8,3,0.018933333456516266
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,3,0.017621333400408428
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,15,0.01759999990463257
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,fp8,7,0.018794666975736618
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,7,0.01781333362062772
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,fp8,15,0.018853332847356796
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,63,0.01810666670401891
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,fp8,63,0.018895999838908512
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,31,0.01794133335351944
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,127,0.01815466706951459
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,fp8,31,0.018768000106016796
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,255,0.021903999149799347
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,fp8,127,0.02161066730817159
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,fp8,255,0.023557332654794056
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,511,0.03376533339420954
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,fp8,511,0.028794666131337483
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,1023,0.04915200173854828
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,fp8,1023,0.04268800218900045
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,fp8,4095,0.10338667035102844
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,4095,0.1365653375784556
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,2047,0.07772266864776611
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,fp8,2047,0.06325866778691609
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,1,0.02437866727511088
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,fp8,3,0.026213333010673523
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,fp8,1,0.026181332767009735
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,3,0.02437866727511088
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,7,0.024383999407291412
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,fp8,7,0.026421333352724712
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,31,0.024645333488782246
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,fp8,31,0.026389333109060924
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,15,0.02456533412138621
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,63,0.02456533412138621
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,fp8,63,0.026191999514897663
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,fp8,15,0.026591998835404713
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,127,0.025434667865435284
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,fp8,127,0.030810666580994923
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,255,0.038202665746212006
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,fp8,255,0.034304000437259674
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,511,0.052069331208864846
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,fp8,511,0.050554667909940086
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,2047,0.1381226678689321
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,1023,0.08186133205890656
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,fp8,2047,0.11404800415039062
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,fp8,1023,0.07087466617425282
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,4095,0.2556053400039673
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,fp8,4095,0.19621866941452026
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,1,0.03774400055408478
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,3,0.03742400060097376
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,fp8,1,0.03989866624275843
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,7,0.03765333443880081
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,fp8,3,0.04031466692686081
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,15,0.037717332442601524
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,fp8,15,0.040378667414188385
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,31,0.037808001041412354
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,fp8,7,0.040991999208927155
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,fp8,31,0.04044266790151596
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,63,0.038058665891488395
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,fp8,63,0.04050133377313614
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,127,0.04409066836039225
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,fp8,127,0.049423997600873314
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,255,0.059450666109720864
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,fp8,255,0.061434666315714516
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,1023,0.1446399986743927
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,fp8,1023,0.12332266569137573
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,511,0.08774399757385254
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,fp8,511,0.08411199847857158
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,2047,0.25895466407140094
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,fp8,2047,0.20950400829315186
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,4095,0.490506649017334
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,fp8,4095,0.3747359911600749
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,1,0.02024000013868014
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,fp8,3,0.021781332790851593
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,7,0.020303999384244282
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,fp8,1,0.021727999051411945
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,3,0.020453333854675293
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,fp8,7,0.02162133405605952
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,15,0.02014933278163274
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,fp8,15,0.021781332790851593
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,31,0.020293333878119785
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,fp8,31,0.021888000269730885
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,63,0.020549333343903225
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,fp8,63,0.021776000658671062
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,127,0.020346666375796
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,fp8,127,0.02439466615517934
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,255,0.02421333392461141
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,fp8,255,0.02619733413060506
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,511,0.037952000896135964
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,fp8,511,0.032560000816980995
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,1023,0.052186667919158936
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,1,0.02647999922434489
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,fp8,1023,0.04640533526738485
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,fp8,1,0.02784000088771184
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,3,0.026704000929991405
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,fp8,3,0.02794666588306427
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,7,0.026714667677879333
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,fp8,7,0.027984000742435455
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,15,0.026778665681680042
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,31,0.026709333062171936
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,fp8,15,0.027978666126728058
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,fp8,31,0.0278613343834877
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,63,0.027306665976842243
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,fp8,63,0.02796799937884013
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,127,0.028501334289709728
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,fp8,127,0.03295466552178065
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,255,0.04055466751257578
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,fp8,255,0.03697066754102707
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,511,0.05555200080076853
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,fp8,511,0.052501335740089417
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,1,0.039887999494870506
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,fp8,1,0.042538667718569435
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,3,0.039690665900707245
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,1023,0.08553600311279297
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,fp8,1023,0.07229333122571309
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,fp8,3,0.042394667863845825
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,7,0.03948266555865606
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,15,0.0402453343073527
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,fp8,7,0.04251733422279358
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,fp8,15,0.04266666869322459
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,31,0.03994666785001755
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,fp8,63,0.04277333120505015
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,127,0.046682665745417275
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,fp8,127,0.05316799879074097
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,fp8,31,0.043050666650136314
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,255,0.062218666076660156
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,63,0.0409706657131513
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,fp8,255,0.06439466774463654
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,1,0.0658240020275116
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,511,0.09123200178146362
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,fp8,511,0.08731733759244283
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,3,0.06585599978764851
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,fp8,1,0.069882666071256
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,fp8,3,0.06975466509660085
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,7,0.06591466565926869
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,1023,0.14669866363207498
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,fp8,15,0.07029866675535838
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,fp8,7,0.07012266914049785
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,fp8,1023,0.1276586651802063
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,31,0.06657599906126659
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,15,0.06592533489068349
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,fp8,31,0.06981333096822102
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,63,0.07173333565394084
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,127,0.07498666644096375
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,fp8,127,0.09186133742332458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,1023,0.27010132869084674
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,fp8,255,0.10620266199111938
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,fp8,63,0.07043200234572093
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,fp8,1023,0.23135999838511148
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,255,0.10048000017801921
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,1,0.03495466709136963
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,fp8,1,0.03666666646798452
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,511,0.1607306698958079
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,fp8,511,0.15065600474675497
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,3,0.03505599995454153
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,fp8,3,0.03656533360481262
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,7,0.03521066655715307
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,15,0.03512533257404963
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,fp8,15,0.036746665835380554
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,fp8,7,0.03664533297220866
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,63,0.03530666728814443
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,fp8,63,0.036517334481080375
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,127,0.03601066768169403
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,fp8,127,0.04083733260631561
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,255,0.04934399823347727
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,fp8,255,0.04600533346335093
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,31,0.03503466645876566
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,1,0.04584533472855886
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,fp8,31,0.036703998843828835
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,511,0.06523733337720235
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,fp8,511,0.06299733122189839
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,fp8,1,0.048432002464930214
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,3,0.04609066744645437
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,7,0.0458186666170756
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,fp8,7,0.04794133206208547
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,fp8,3,0.048058668772379555
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,15,0.04614933331807455
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,fp8,15,0.048154667019844055
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,31,0.0467199981212616
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,fp8,31,0.0489279975493749
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,63,0.04750399788220724
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,fp8,63,0.04833066463470459
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,127,0.05527999997138977
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,fp8,127,0.059621334075927734
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,255,0.06837333242098491
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,fp8,255,0.0721973329782486
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,1,0.07335466643174489
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,511,0.10053867101669312
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,fp8,1,0.07820799946784973
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,fp8,511,0.09499200185139973
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,3,0.07313066720962524
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,fp8,3,0.07810666660467784
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,7,0.07354133327802022
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,fp8,7,0.07817066709200542
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,15,0.07388266424338023
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,fp8,15,0.07924266656239827
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,31,0.07513600091139476
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,fp8,63,0.08052800099054973
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,63,0.0798933357000351
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,127,0.08283733328183492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,fp8,31,0.07871466875076294
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,fp8,127,0.10050666332244873
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,255,0.10994133353233337
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,fp8,255,0.11726400256156921
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,3,0.12566399574279785
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,1,0.1254026691118876
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,fp8,1,0.13211199641227722
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,511,0.16973867019017538
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,fp8,511,0.16375999649365744
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,fp8,3,0.13221866885821024
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,7,0.12544000148773193
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,15,0.12769599755605063
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,fp8,15,0.13201600313186646
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,31,0.131221334139506
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,fp8,7,0.13379200299580893
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,fp8,63,0.13832533359527588
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,fp8,31,0.13474133610725403
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,63,0.13355200489362082
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,127,0.1395840048789978
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,fp8,127,0.1721173326174418
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,1,0.011215999722480774
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,7,0.011215999722480774
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,1,0.011989332735538483
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,7,0.012133333832025528
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,15,0.010970667004585266
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,fp8,255,0.19966934124628702
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,15,0.012149333953857422
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,31,0.011002667248249054
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,3,0.012138667205969492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,63,0.011231999844312668
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,63,0.012096000214417776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,127,0.011007999380429586
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,255,0.1871253252029419
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,127,0.013343999783198038
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,255,0.012191999703645706
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,255,0.01431999976436297
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,3,0.011114666859308878
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,511,0.01569066693385442
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,1023,0.015669333438078564
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,1023,0.01754666616519292
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,2047,0.019999999552965164
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,2047,0.02257599929968516
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,4095,0.02869333326816559
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,4095,0.03259200106064478
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,8191,0.04729066789150238
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,8191,0.051632001996040344
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,fp8,511,0.28941865762074787
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,16383,0.08272533118724823
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,16383,0.0965119997660319
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,32767,0.15311466654141745
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,32767,0.18263999621073404
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,65535,0.35097599029541016
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,1,0.011168000598748526
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,1,0.011957333733638128
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,3,0.01128000020980835
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,3,0.012015999605258306
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,31,0.012069333344697952
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,7,0.011205332974592844
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,7,0.011839999506870905
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,15,0.01129066695769628
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,511,0.013818666338920593
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,15,0.011786667009194693
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,31,0.011247999966144562
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,31,0.011989332735538483
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,63,0.011322667201360067
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,511,0.31004265944163006
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,63,0.011850666254758835
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,127,0.011306667079528173
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,65535,0.2922933300336202
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,127,0.013114667187134424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,511,0.01573333392540614
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,255,0.01240533341964086
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,255,0.015530666957298914
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,1023,0.016352000335852306
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,1023,0.018650667121013004
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,2047,0.02073066681623459
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,2047,0.023039999107519787
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,4095,0.029130667448043823
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,4095,0.03292266776164373
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,16383,0.0839359958966573
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,8191,0.04846400022506714
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,8191,0.05322133501370748
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,511,0.014912000546852747
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,16383,0.09328533212343852
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,32767,0.1534293293952942
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,32767,0.17311465740203857
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,65535,0.2925706704457601
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,1,0.011781333635250727
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,65535,0.3330026666323344
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,1,0.013530666629473368
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,3,0.011637333780527115
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,15,0.012042666474978128
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,3,0.013056000073750814
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,7,0.012015999605258306
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,7,0.01310933381319046
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,15,0.012869333227475485
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,63,0.01312000056107839
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,31,0.011786667009194693
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,31,0.012789333860079447
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,255,0.013274667163689932
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,63,0.012053333222866058
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,127,0.014831999937693277
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,127,0.014725333700577417
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,255,0.015477333217859268
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,511,0.014901333798964819
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,511,0.016650666793187458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,1023,0.016421332955360413
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,1023,0.018768000106016796
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,2047,0.022218666970729828
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,2047,0.023728000621000927
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,16383,0.112527996301651
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,4095,0.03398400048414866
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,4095,0.03450666616360346
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,8191,0.06234133243560791
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,8191,0.05835199852784475
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,16383,0.10116799672444661
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,32767,0.16827734311421713
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,32767,0.18366400400797525
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,65535,0.33345599969228107
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,1,0.011898666620254517
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,1,0.012810666114091873
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,65535,0.3518666823705037
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,3,0.012133333832025528
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,3,0.013178666432698568
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,7,0.01179733375708262
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,7,0.013104000439246496
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,15,0.01184533288081487
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,15,0.01333333303531011
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,31,0.012026666353146235
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,31,0.013269333789745966
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,63,0.012186666329701742
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,63,0.013114667187134424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,127,0.01202133297920227
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,127,0.014671999961137772
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,255,0.013669333110253016
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,255,0.016143999993801117
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,511,0.015173333386580149
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,511,0.01699200024207433
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,1023,0.01728533332546552
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,1023,0.01926400015751521
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,2047,0.030282666285832722
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,2047,0.024746666351954143
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,4095,0.04557866851488749
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,4095,0.038719999293486275
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,8191,0.0759200006723404
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,8191,0.06028266747792562
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,16383,0.1335093379020691
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,16383,0.10140800476074219
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,32767,0.2584373354911804
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,32767,0.185370663801829
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,65535,0.5019413232803345
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,65535,0.35676801204681396
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,1,0.03812266637881597
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,fp8,1,0.03932266682386398
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,fp8,3,0.039120001097520195
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,3,0.03879466652870178
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,fp8,7,0.03922666609287262
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,7,0.038634667793909706
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,fp8,15,0.03898133337497711
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,15,0.03832533210515976
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,31,0.03830400109291077
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,fp8,31,0.03895466774702072
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,63,0.040778666734695435
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,fp8,63,0.03910933434963226
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,127,0.045893331368764244
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,fp8,127,0.04931733508904775
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,255,0.05719999969005585
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,fp8,255,0.06329600016276042
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,1,0.06182933350404104
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,fp8,1,0.062447999914487205
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,3,0.06180266539255778
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,fp8,3,0.06277333199977875
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,7,0.06205333272616068
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,fp8,7,0.06274133423964183
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,15,0.061978667974472046
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,fp8,15,0.06281066437562306
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,31,0.06384000182151794
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,fp8,31,0.06268799801667531
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,63,0.06983999907970428
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,fp8,63,0.06480533381303151
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,127,0.07332266867160797
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,fp8,127,0.08473599950472514
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,255,0.09573866923650105
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,1,0.11581333478291829
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,fp8,255,0.10223467151323955
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,fp8,1,0.11454400420188904
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,3,0.11571199695269267
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,fp8,3,0.11410666505495708
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,7,0.11612799763679504
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,fp8,7,0.11296000083287557
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,fp8,15,0.11395200093587239
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,fp8,31,0.11926933129628499
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,15,0.12143466869990031
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,63,0.12743467092514038
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,31,0.12548266847928366
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,fp8,63,0.12468266487121582
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,127,0.1325706640879313
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,fp8,127,0.1572160025437673
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,1,0.21227733294169107
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,fp8,1,0.20671466986338297
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,3,0.2111306587855021
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,255,0.17482133706410727
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,fp8,255,0.1875093380610148
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,fp8,3,0.20676799615224203
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,15,0.2293813427289327
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,7,0.22235733270645142
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,fp8,15,0.21439466873804727
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,fp8,7,0.20594666401545206
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,31,0.2285333275794983
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,fp8,63,0.2222879926363627
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,63,0.23357333739598593
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,fp8,31,0.22165334224700928
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,127,0.23996800184249878
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,1,0.06513066589832306
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,fp8,127,0.2916106581687927
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,255,0.32501866420110065
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,fp8,255,0.3412533203760783
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,fp8,1,0.065461332599322
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,fp8,3,0.06621333460013072
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,3,0.06572266419728597
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,7,0.06569600105285645
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,fp8,7,0.06604800124963124
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,15,0.06790933509667714
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,fp8,31,0.06710933148860931
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,63,0.07328000168005626
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,fp8,15,0.06587733328342438
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,fp8,63,0.07411199808120728
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,31,0.07198399802049
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,127,0.07625600198904674
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,fp8,127,0.0893440047899882
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,1,0.1123413344224294
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,fp8,1,0.11286399761835735
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,3,0.11204266548156738
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,fp8,3,0.11313600341478984
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,7,0.11353066563606262
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,fp8,7,0.11268267035484314
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,15,0.1172213355700175
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,31,0.12213333447774251
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,fp8,15,0.11329066753387451
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,fp8,31,0.11716799934705098
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,63,0.12452266613642375
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,fp8,63,0.12477866808573405
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,1,0.2202933430671692
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,127,0.13115732868512472
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,fp8,1,0.2148373325665792
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,fp8,127,0.15306666493415833
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,3,0.2199359933535258
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,fp8,3,0.21443732579549155
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,7,0.23438400030136108
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,fp8,7,0.21570666631062826
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,15,0.23789334297180176
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,31,0.23917333285013834
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,fp8,15,0.2281173268953959
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,fp8,31,0.2304746707280477
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,63,0.23919467131296793
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,fp8,63,0.23303999503453574
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,1,0.41674665609995526
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,127,0.24897066752115884
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,fp8,127,0.30082132418950397
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,fp8,1,0.4014773368835449
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,3,0.4439306656519572
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,fp8,3,0.40121599038441974
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,7,0.44153066476186115
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,fp8,7,0.41969064871470135
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,15,0.4444799820582072
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,fp8,31,0.4277600049972534
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,63,0.4530773162841797
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,fp8,63,0.4270506699879964
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,31,0.4456319808959961
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,fp8,15,0.4253013531366984
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,1,0.01138666644692421
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,7,0.012389333297808966
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,15,0.011648000528415045
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,7,0.011461333682139715
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,15,0.012421333541472753
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,31,0.011498666057984034
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,31,0.012293333808581034
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,63,0.011349332829316458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,63,0.012389333297808966
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,127,0.01137599969903628
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,127,0.01358933374285698
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,3,0.012453333785136541
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,255,0.012773333738247553
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,255,0.014362666755914688
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,511,0.016917333006858826
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,511,0.01471466695268949
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,1,0.012442667037248611
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,3,0.011354666203260422
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,1023,0.019296000401178997
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,1023,0.021701333423455555
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,127,0.46242133776346844
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,2047,0.027823999524116516
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,2047,0.031712000568707786
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,8191,0.09321066737174988
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,4095,0.045642669002215065
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,4095,0.050570666790008545
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,8191,0.0828000009059906
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,32767,0.3507893482844035
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,16383,0.15194666385650635
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,32767,0.29205866654713947
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,1,0.012149333953857422
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,1,0.012890666723251343
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,3,0.012149333953857422
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,3,0.012543999900420507
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,7,0.012026666353146235
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,7,0.01250133290886879
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,15,0.01209066684047381
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,15,0.012565333396196365
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,16383,0.1820853352546692
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,31,0.012138667205969492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,31,0.012613333761692047
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,63,0.012117333710193634
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,63,0.012602667013804117
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,127,0.01232533281048139
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,127,0.01402666668097178
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,255,0.013536000003417334
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,255,0.014645333091417948
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,511,0.01540800059835116
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,511,0.017173333714405697
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,1023,0.020021333048741024
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,fp8,127,0.5686560074488322
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,1023,0.021984001000722248
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,2047,0.029333333174387615
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,2047,0.031925333042939506
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,4095,0.04814933240413666
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,4095,0.052058666944503784
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,8191,0.08304533362388611
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,8191,0.09427733222643535
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,16383,0.15388799707094827
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,32767,0.2924426595369975
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,1,0.012159999459981918
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,32767,0.3285173376401265
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,1,0.012991999586423239
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,3,0.011829332758982977
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,3,0.01303999995191892
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,7,0.01228800043463707
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,7,0.013061333447694778
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,15,0.012229333321253458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,15,0.013072000195582708
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,16383,0.1726293365160624
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,31,0.012186666329701742
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,31,0.013280000537633896
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,63,0.012181332955757776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,63,0.0136266661187013
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,127,0.012165332833925882
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,127,0.01469333345691363
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,255,0.013717333475748697
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,255,0.015002666662136713
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,511,0.015642666568358738
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,511,0.018122666825850803
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,1023,0.020586666961510975
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,1023,0.023002666731675465
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,2047,0.03275199979543686
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,2047,0.03499733408292135
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,4095,0.0525546669960022
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,4095,0.05710400144259135
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,8191,0.08995733658472697
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,8191,0.09921600421269734
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,16383,0.17107733090718588
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,16383,0.1832480033238729
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,32767,0.29730133215586346
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,1,0.012266666938861212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,32767,0.35155200958251953
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,1,0.013162666310866674
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,3,0.012186666329701742
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,3,0.01328533391157786
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,7,0.012042666474978128
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,7,0.01351999988158544
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,15,0.0124746672809124
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,15,0.013317332913478216
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,31,0.012229333321253458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,31,0.01357866699496905
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,63,0.012357333054145178
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,63,0.013562666873137156
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,127,0.012709333250919977
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,127,0.015216000378131866
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,255,0.014346666634082794
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,255,0.015834666788578033
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,511,0.01616000011563301
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,511,0.01838933303952217
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,1023,0.02826666583617528
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,1023,0.024336000283559162
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,2047,0.04397333165009817
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,2047,0.03798400113979975
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,8191,0.13491732875506082
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,4095,0.07526400188604991
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,4095,0.05816000203291575
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,8191,0.10525332887967427
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,16383,0.25334932406743366
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,16383,0.19248000780741373
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,1,0.011717333147923151
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,1,0.012784000486135483
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,32767,0.49742400646209717
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,32767,0.353770653406779
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,3,0.011829332758982977
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,3,0.013503999759753546
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,7,0.01181866725285848
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,7,0.013258667041858038
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,15,0.011920000116030375
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,15,0.012682666381200155
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,31,0.011994666109482447
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,127,0.012106666962305704
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,31,0.012650666137536367
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,63,0.011930666863918304
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,63,0.013221333424250284
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,127,0.014138666292031607
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,255,0.01328533391157786
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,255,0.014837333311637243
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,511,0.01553600033124288
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,511,0.017535999417304993
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,1023,0.019706666469573975
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,1023,0.022698665658632915
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,2047,0.028431999186674755
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,2047,0.0322826678554217
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,4095,0.048063998421033226
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,4095,0.05146666864554087
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,8191,0.08301333089669545
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,8191,0.09968533118565877
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,16383,0.1539413332939148
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,16383,0.1846933364868164
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,1,0.012256000190973282
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,1,0.012991999586423239
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,3,0.012309333930412928
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,15,0.013317332913478216
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,3,0.012938667088747025
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,7,0.011877333124478659
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,7,0.013381333400805792
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,15,0.011994666109482447
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,31,0.012058666596810022
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,31,0.012975999464591345
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,63,0.012282667060693106
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,63,0.013264000415802002
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,127,0.012421333541472753
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,127,0.014458666245142618
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,255,0.013573333621025085
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,255,0.015402667224407196
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,511,0.015626666446526844
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,511,0.017717332889636356
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,1023,0.0204373337328434
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,1023,0.02317333221435547
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,2047,0.03253333270549774
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,2047,0.03281066566705704
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,4095,0.053397332628568016
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,4095,0.05773866673310598
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,16383,0.18425599733988443
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,8191,0.09361066420873006
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,8191,0.09967999656995137
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,16383,0.15619200468063354
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,1,0.012213333199421564
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,1,0.013541333377361298
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,3,0.012543999900420507
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,3,0.013573333621025085
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,7,0.012453333785136541
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,7,0.013637332866589228
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,15,0.012293333808581034
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,15,0.013637332866589228
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,31,0.012485332787036896
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,31,0.013728000223636627
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,63,0.01259200026591619
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,63,0.01368533323208491
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,127,0.012821332861979803
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,127,0.015210667004187902
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,255,0.014298666268587112
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,255,0.015813333292802174
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,511,0.016058667252461117
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,511,0.018458666900793713
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,1023,0.028746667007605236
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,1023,0.023946667710940044
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,2047,0.0440586656332016
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,2047,0.03772266705830892
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,4095,0.07366933425267537
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,4095,0.058431997895240784
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,8191,0.13405866424242655
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,8191,0.10100799798965454
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,1,0.011136000355084738
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,16383,0.2531413237253825
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,16383,0.1850186586380005
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,1,0.012175999581813812
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,3,0.011168000598748526
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,3,0.012138667205969492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,7,0.011077333241701126
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,7,0.01211200033624967
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,15,0.011237333218256632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,15,0.01219733307758967
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,31,0.011007999380429586
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,31,0.012181332955757776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,63,0.011141333729028702
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,63,0.012517333030700684
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,127,0.011152000476916632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,127,0.014554666976133982
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,255,0.013264000415802002
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,255,0.014192000031471252
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,511,0.013381333400805792
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,511,0.01444799949725469
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,1023,0.013701333353916803
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,1023,0.01482133318980535
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,2047,0.014671999961137772
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,2047,0.015754666179418564
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,4095,0.01568000018596649
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,4095,0.016789333273967106
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,8191,0.018112000077962875
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,8191,0.019519999623298645
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,16383,0.0223786657055219
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,16383,0.02492266645034154
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,32767,0.032272001107533775
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,1,0.012346666306257248
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,3,0.011002667248249054
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,32767,0.03501333296298981
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,65535,0.05555200080076853
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,65535,0.05064000189304352
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,1,0.011125333607196808
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,3,0.012128000458081564
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,7,0.010949333508809408
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,7,0.012367999802033106
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,15,0.011045332998037338
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,15,0.012416000167528788
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,31,0.011168000598748526
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,31,0.012341332932313284
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,63,0.011178666104873022
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,63,0.012261333564917246
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,127,0.01118933285276095
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,127,0.014549333602190018
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,255,0.013376000026861826
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,255,0.01413333291808764
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,511,0.013248000293970108
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,511,0.014405333747466406
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,1023,0.013818666338920593
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,1023,0.015066667149464289
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,2047,0.014912000546852747
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,2047,0.016415999581416447
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,4095,0.016714667280515034
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,4095,0.017312000195185345
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,8191,0.018826667219400406
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,8191,0.020202666521072388
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,16383,0.023178666830062866
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,16383,0.025685332715511322
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,32767,0.033813332517941795
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,32767,0.03589866558710734
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,65535,0.05358933409055074
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,65535,0.05730666716893514
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,1,0.01108266661564509
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,1,0.012234666695197424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,3,0.011194666226704916
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,3,0.012042666474978128
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,7,0.01145600030819575
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,7,0.012367999802033106
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,15,0.01129066695769628
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,127,0.011445333560307821
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,15,0.012245333443085352
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,31,0.011194666226704916
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,31,0.012272000312805176
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,63,0.011312000453472137
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,63,0.012240000069141388
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,511,0.015397333850463232
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,127,0.014501333236694336
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,255,0.013642666240533194
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,255,0.01471466695268949
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,511,0.013717333475748697
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,1023,0.014458666245142618
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,1023,0.015562667200962702
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,2047,0.016063999384641647
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,2047,0.01743999992807706
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,4095,0.01699200024207433
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,4095,0.017680000513792038
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,8191,0.019226666539907455
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,8191,0.020303999384244282
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,16383,0.02348800003528595
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,16383,0.026176000634829204
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,32767,0.036357333262761436
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,32767,0.03673599908749262
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,65535,0.05794133245944977
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,65535,0.06229866544405619
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,1,0.010847999403874079
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,1,0.012117333710193634
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,3,0.011050666371981302
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,3,0.011941333611806234
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,15,0.012074666718641916
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,7,0.011114666859308878
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,7,0.012122667084137598
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,15,0.011215999722480774
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,31,0.011253333340088526
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,31,0.01231466606259346
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,63,0.011178666104873022
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,63,0.01209066684047381
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,127,0.011055999745925268
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,127,0.014538666854302088
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,255,0.01360000049074491
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,255,0.01434133326013883
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,511,0.013445333888133367
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,511,0.014368000129858652
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,1023,0.014709333578745524
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,1023,0.015504000087579092
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,2047,0.015978666643301647
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,2047,0.01682666689157486
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,4095,0.017952000101407368
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,4095,0.019648000597953796
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,8191,0.02141333371400833
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,8191,0.023813332120577495
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,16383,0.02977066735426585
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,16383,0.0342399999499321
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,32767,0.04852266609668732
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,32767,0.05320533116658529
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,65535,0.0844053328037262
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,65535,0.09318400422732036
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,3,0.011328000575304031
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,1,0.011146667102972666
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,1,0.012106666962305704
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,3,0.012469333906968435
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,7,0.012282667060693106
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,7,0.011231999844312668
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,15,0.011418666690587997
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,15,0.012223999947309494
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,31,0.01129066695769628
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,31,0.012117333710193634
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,63,0.011328000575304031
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,63,0.01246400053302447
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,127,0.01128000020980835
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,127,0.014490666488806406
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,255,0.013503999759753546
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,255,0.014538666854302088
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,511,0.013733333597580591
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,511,0.015141333142916361
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,1023,0.014954666296641031
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,1023,0.016565332810084026
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,2047,0.01658133293191592
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,2047,0.017237332959969837
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,4095,0.018768000106016796
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,4095,0.020080000162124634
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,8191,0.02222399910291036
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,8191,0.02463999887307485
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,16383,0.030954666435718536
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,16383,0.03502399971087774
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,32767,0.05053866902987162
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,32767,0.05504000186920166
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,65535,0.08542933066685994
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,65535,0.10166933139165242
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,1,0.011482667177915573
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,1,0.012367999802033106
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,3,0.011306667079528173
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,3,0.012634667257467905
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,7,0.011567999919255575
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,7,0.01240533341964086
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,15,0.011317333827416102
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,15,0.012618667135636011
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,31,0.011637333780527115
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,31,0.012309333930412928
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,63,0.011237333218256632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,63,0.012730666746695837
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,127,0.011472000430027643
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,127,0.01481066644191742
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,255,0.013594667116800943
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,255,0.015344000111023584
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,511,0.014394666999578476
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,511,0.015615999698638916
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,1023,0.015925332903862
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,1023,0.017050666113694508
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,2047,0.016688000410795212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,2047,0.01759999990463257
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,4095,0.019050666441520054
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,4095,0.020309332758188248
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,8191,0.022495999932289124
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,8191,0.025226667523384094
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,16383,0.03517866631348928
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,16383,0.03579733272393545
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,32767,0.05113600194454193
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,32767,0.058970664938290916
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,65535,0.09465066591898601
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,65535,0.1023520032564799
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,1,0.01301866645614306
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,fp8,1,0.013557333499193192
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,3,0.012965332716703415
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,fp8,3,0.013493333011865616
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,7,0.012597333639860153
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,fp8,7,0.013557333499193192
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,15,0.01292266696691513
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,fp8,15,0.013733333597580591
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,31,0.012703999876976013
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,fp8,31,0.013408000270525614
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,63,0.01292266696691513
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,fp8,63,0.013653332988421122
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,127,0.012858666479587555
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,fp8,127,0.014778666198253632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,255,0.014730667074521383
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,fp8,511,0.01830400029818217
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,fp8,255,0.015552000453074774
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,511,0.016176000237464905
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,1023,0.0210506667693456
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,fp8,1023,0.022858666876951855
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,2047,0.033813332517941795
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,fp8,2047,0.03283733377854029
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,4095,0.05440000196297964
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,fp8,4095,0.057477335135142006
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,8191,0.11132267117500305
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,1,0.013061333447694778
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,fp8,8191,0.10155733426411946
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,fp8,1,0.01392000044385592
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,3,0.01312000056107839
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,fp8,3,0.013898666948080063
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,fp8,7,0.013866666704416275
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,7,0.013210666676362356
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,15,0.013023999830087027
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,fp8,15,0.014085333794355392
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,31,0.013050666699806849
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,fp8,31,0.014192000031471252
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,63,0.013210666676362356
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,fp8,63,0.014096000542243322
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,127,0.01322666679819425
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,255,0.014549333602190018
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,fp8,127,0.015541333705186844
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,fp8,255,0.016021333634853363
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,511,0.016970666746298473
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,fp8,511,0.018874666343132656
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,1023,0.029690665503342945
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,fp8,1023,0.023754666248957317
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,2047,0.04450133442878723
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,fp8,2047,0.03841600070397059
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,4095,0.0745119998852412
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,fp8,4095,0.05898666878541311
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,8191,0.1360266705354055
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,fp8,8191,0.10222400228182475
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,1,0.0170666662355264
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,fp8,1,0.018138666947682697
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,3,0.01666133354107539
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,fp8,3,0.017898666361967724
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,7,0.017312000195185345
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,fp8,7,0.01814933369557063
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,15,0.017322666943073273
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,63,0.017386666188637417
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,fp8,15,0.01806933308641116
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,31,0.017045332739750545
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,fp8,31,0.018165333817402523
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,fp8,63,0.018288000176350277
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,127,0.017909333109855652
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,fp8,127,0.02143466720978419
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,255,0.02092266579469045
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,fp8,255,0.022458667556444805
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,511,0.0322773332397143
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,fp8,511,0.027562665442625683
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,1023,0.047135998805363975
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,fp8,1023,0.04224533339341482
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,2047,0.07674666742483775
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,fp8,2047,0.0639519989490509
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,4095,0.13583466410636902
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,fp8,4095,0.10502933462460835
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,3,0.01109333336353302
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,1,0.010816000401973724
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,8191,0.25176533063252765
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,fp8,8191,0.1904266675313314
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,3,0.011727999895811081
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,1,0.011829332758982977
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,7,0.01071999967098236
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,7,0.011546666423479715
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,15,0.011157333850860596
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,15,0.011754666765530905
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,31,0.011407999942700068
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,31,0.011637333780527115
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,63,0.010741333166758219
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,63,0.011829332758982977
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,127,0.010666667173306147
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,127,0.013994666437307993
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,255,0.013338666409254074
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,255,0.014149333039919535
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,511,0.014245333770910898
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,511,0.015322666615247726
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,1023,0.015568000574906668
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,1023,0.016442666451136272
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,2047,0.016965333372354507
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,2047,0.018901333212852478
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,4095,0.021183999876181286
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,4095,0.023530667026837666
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,8191,0.02985599885384242
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,32767,0.08413867155710857
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,8191,0.03339199970165888
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,16383,0.048197334011395775
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,16383,0.053727999329566956
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,32767,0.09735999504725139
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,65535,0.15263467033704123
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,65535,0.18457067012786865
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,1,0.010821333775917688
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,1,0.0120319997270902
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,3,0.010992000500361124
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,3,0.011802667131026586
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,15,0.011930666863918304
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,7,0.011018666128317514
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,7,0.01180800050497055
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,15,0.011152000476916632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,31,0.011087999989589056
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,31,0.011936000237862269
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,63,0.010751999914646149
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,63,0.011648000528415045
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,127,0.011183999478816986
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,127,0.014416000495354334
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,255,0.01350933313369751
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,255,0.015002666662136713
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,511,0.014565333724021912
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,511,0.01642666632930438
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,1023,0.016048000504573185
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,1023,0.017045332739750545
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,2047,0.017509333789348602
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,2047,0.019402666638294857
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,4095,0.021850667893886566
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,4095,0.024336000283559162
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,8191,0.030346666773160298
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,8191,0.03437866767247518
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,16383,0.04944533109664917
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,16383,0.056736002365748085
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,32767,0.08556800087292989
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,32767,0.10081600149472554
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,65535,0.15552533666292825
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,65535,0.184063990910848
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,1,0.011413333316644033
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,1,0.012138667205969492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,3,0.01128000020980835
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,3,0.012229333321253458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,7,0.011407999942700068
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,7,0.012416000167528788
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,15,0.011359999577204386
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,15,0.012506666282812754
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,31,0.011349332829316458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,31,0.01209066684047381
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,255,0.015018666783968607
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,511,0.015637333194414776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,63,0.011354666203260422
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,63,0.012298667182525
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,127,0.011535999675591787
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,127,0.014912000546852747
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,255,0.01413333291808764
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,511,0.01670933390657107
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,1023,0.01664000004529953
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,1023,0.017317333569129307
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,2047,0.017450666675964992
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,2047,0.019797333826621372
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,4095,0.022469334304332733
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,4095,0.02459733436505
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,8191,0.03387733300526937
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,8191,0.03489066660404205
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,16383,0.051114668448766075
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,16383,0.05932266513506571
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,32767,0.08611200253168742
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,32767,0.10161599516868591
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,65535,0.17314666509628296
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,65535,0.18362132708231607
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,1,0.014346666634082794
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,fp8,1,0.015386667102575302
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,3,0.014138666292031607
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,fp8,3,0.015306666493415833
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,7,0.013866666704416275
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,fp8,7,0.015253332753976187
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,15,0.013989333063364029
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,fp8,15,0.015189333508412043
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,31,0.014335999886194864
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,fp8,31,0.015413332730531693
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,63,0.014277332772811254
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,fp8,63,0.015429332852363586
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,127,0.014442666123310724
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,fp8,127,0.018357332795858383
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,255,0.016085332880417507
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,fp8,255,0.017423999806245167
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,511,0.01930133377512296
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,fp8,511,0.020138667275508244
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,1023,0.030623999734719593
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,fp8,1023,0.025253333151340485
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,2047,0.04621333380540212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,fp8,2047,0.04021333406368891
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,1,0.017621333400408428
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,fp8,1,0.019071999937295914
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,4095,0.07625066737333934
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,3,0.017610666652520496
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,fp8,3,0.018933333456516266
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,fp8,4095,0.061146666606267296
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,7,0.017786666750907898
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,fp8,7,0.01905599981546402
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,15,0.017898666361967724
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,31,0.017701332767804463
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,fp8,15,0.01922133316596349
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,fp8,31,0.018976000448067982
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,fp8,63,0.018901333212852478
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,127,0.01817600056529045
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,63,0.01794133335351944
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,fp8,127,0.021664001047611237
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,255,0.02165866643190384
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,fp8,255,0.023498666783173878
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,511,0.032618666688601174
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,fp8,511,0.028912000358104706
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,1023,0.049733335773150124
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,fp8,1023,0.043007999658584595
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,4095,0.13637333114941916
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,2047,0.07829866806666057
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,1,0.024421334266662598
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,fp8,4095,0.10673066973686218
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,fp8,1,0.0262773334980011
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,fp8,2047,0.06497066716353099
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,3,0.02443733314673106
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,fp8,3,0.026767998933792114
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,fp8,7,0.026133333643277485
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,15,0.02455466737349828
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,fp8,15,0.026309333741664886
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,fp8,31,0.026234666506449383
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,7,0.024469333390394848
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,fp8,63,0.026335999369621277
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,63,0.025173333783944447
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,127,0.025439999997615814
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,fp8,127,0.030896000564098358
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,255,0.03783999880154928
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,fp8,255,0.034517332911491394
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,31,0.02481599897146225
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,511,0.05111999809741974
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,fp8,511,0.05012799799442291
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,2047,0.13922133048375449
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,fp8,2047,0.11336533228556316
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,fp8,1023,0.07106666763623555
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,4095,0.25523199637730914
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,fp8,4095,0.19611199696858725
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,1023,0.08055466910203297
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,1,0.019754666835069656
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,fp8,1,0.021397332350413006
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,3,0.019973333925008774
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,fp8,3,0.021551998953024547
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,fp8,7,0.021690666675567627
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,15,0.019920000185569126
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,7,0.020037333170572918
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,31,0.019551999866962433
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,fp8,31,0.02161066730817159
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,63,0.020031999796628952
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,fp8,63,0.02164799968401591
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,127,0.020319999506076176
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,fp8,127,0.024293333292007446
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,fp8,15,0.021856000026067097
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,255,0.023872000475724537
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,fp8,255,0.025807999074459076
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,511,0.035877334574858345
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,fp8,511,0.03160000095764796
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,1023,0.052255998055140175
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,fp8,1023,0.047007997830708824
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,1,0.026320000489552815
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,fp8,1,0.02888533224662145
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,3,0.02622399975856145
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,fp8,3,0.02811199923356374
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,7,0.026416001220544178
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,fp8,7,0.02828266719977061
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,31,0.026613332331180573
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,fp8,31,0.02881066749493281
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,63,0.02657066782315572
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,fp8,63,0.028175999720891316
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,127,0.027056001126766205
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,fp8,15,0.02842666705449422
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,15,0.026517334083716076
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,fp8,127,0.03309866786003113
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,255,0.04054400076468786
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,fp8,255,0.03940266619126002
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,511,0.055776000022888184
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,fp8,511,0.05354666709899902
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,1,0.03925866633653641
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,fp8,1,0.04205333193143209
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,1023,0.0851093331972758
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,fp8,1023,0.0738453318675359
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,3,0.039290666580200195
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,fp8,3,0.04322666426499685
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,7,0.039349332451820374
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,15,0.03965866565704346
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,fp8,7,0.042223999897638954
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,fp8,15,0.042266666889190674
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,63,0.04022400081157684
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,fp8,63,0.043136000633239746
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,127,0.04637333254019419
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,fp8,127,0.051221330960591636
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,31,0.03956266740957896
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,fp8,31,0.042768001556396484
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,fp8,255,0.0639466643333435
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,255,0.06099733213583628
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,511,0.08984000484148662
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,fp8,511,0.08796266714731853
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,1,0.03405333310365677
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,1023,0.14639467000961304
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,fp8,1023,0.12654399871826172
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,fp8,1,0.0359253336985906
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,3,0.033573334415753685
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,fp8,7,0.0359199990828832
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,7,0.034048000971476235
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,fp8,3,0.03613866617282232
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,fp8,31,0.03570666660865148
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,31,0.034346667428811394
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,63,0.034330666065216064
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,fp8,63,0.03600533306598663
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,127,0.03526400029659271
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,fp8,15,0.03573333223660787
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,15,0.0344106654326121
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,fp8,127,0.040037333965301514
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,255,0.047151997685432434
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,fp8,255,0.04451733330885569
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,511,0.06318933268388112
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,fp8,511,0.061834668119748436
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,1,0.04598933458328247
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,fp8,1,0.04836266736189524
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,3,0.045653333266576133
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,fp8,3,0.04860800007979075
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,7,0.04560000201066335
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,fp8,7,0.04844266672929128
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,15,0.046122665206591286
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,31,0.046223998069763184
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,fp8,31,0.04858666658401489
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,fp8,15,0.048656001687049866
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,63,0.04710933566093445
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,fp8,63,0.04862933357556661
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,127,0.054373333851496376
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,fp8,127,0.05783999959627787
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,255,0.06870399912198384
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,fp8,255,0.07285333176453908
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,1,0.07253866891066234
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,511,0.09935466448465984
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,fp8,511,0.09613333145777385
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,fp8,1,0.07796800136566162
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,3,0.07239466905593872
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,fp8,3,0.07791999975840251
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,15,0.07269333302974701
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,fp8,7,0.07769066592057546
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,7,0.0738613357146581
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,fp8,31,0.07791466514269511
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,63,0.07956266899903615
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,fp8,15,0.078015998005867
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,fp8,63,0.07881600161393483
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,31,0.07426133255163829
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,1,0.011242666592200598
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,127,0.08347200353940327
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,1,0.011989332735538483
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,fp8,127,0.10035733381907146
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,3,0.011237333218256632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,7,0.011077333241701126
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,3,0.012063999970753988
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,15,0.011061333119869232
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,15,0.012128000458081564
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,fp8,255,0.11619733770688374
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,255,0.10724799831708272
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,7,0.012037333101034164
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,31,0.011061333119869232
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,31,0.012202666451533636
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,63,0.010944000134865442
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,127,0.013173333058754602
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,511,0.16826667388280234
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,255,0.012144000579913458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,255,0.015418666104475657
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,fp8,511,0.16259732842445374
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,511,0.014783999572197596
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,63,0.012096000214417776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,511,0.01632533346613248
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,127,0.01129066695769628
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,1023,0.01894933357834816
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,2047,0.02130666623512904
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,4095,0.029909332593282063
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,4095,0.03356799980004629
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,8191,0.048207998275756836
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,1023,0.01684800038735072
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,8191,0.05332799752553304
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,16383,0.08364267150561015
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,2047,0.02372266600529353
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,16383,0.09469333291053772
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,32767,0.15428800384203592
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,32767,0.18513067563374838
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,65535,0.29315733909606934
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,1,0.011503999431928
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,65535,0.3557120164235433
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,1,0.012389333297808966
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,3,0.011253333340088526
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,3,0.01228800043463707
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,7,0.01129066695769628
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,7,0.012298667182525
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,15,0.011136000355084738
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,15,0.01227733368674914
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,31,0.01129066695769628
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,31,0.012266666938861212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,63,0.011183999478816986
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,63,0.012122667084137598
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,255,0.016447999825080235
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,127,0.011274666835864386
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,127,0.013306666165590286
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,255,0.012693333129088083
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,511,0.015418666104475657
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,511,0.01710933322707812
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,2047,0.024314666787783306
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,1023,0.01754133279124896
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,1023,0.01931200052301089
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,2047,0.021898667017618816
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,4095,0.03091199944416682
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,4095,0.03442666679620743
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,8191,0.050197333097457886
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,8191,0.05374933282534281
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,16383,0.08559466401735942
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,16383,0.10152000188827515
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,32767,0.15445866187413534
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,32767,0.18585066000620523
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,65535,0.2948319911956787
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,1,0.011424000064531961
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,65535,0.35259731610616046
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,1,0.012928000340859095
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,3,0.0116799995303154
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,3,0.012789333860079447
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,7,0.012015999605258306
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,7,0.012831999609867731
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,15,0.011765333513418833
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,15,0.012698666503032049
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,31,0.011786667009194693
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,31,0.012821332861979803
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,255,0.013781332721312841
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,63,0.011754666765530905
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,63,0.01320533330241839
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,127,0.011882666498422623
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,127,0.01431999976436297
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,255,0.01711999997496605
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,511,0.01613866661985715
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,511,0.01762666677435239
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,1023,0.017765333255132038
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,1023,0.01988799994190534
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,2047,0.022517333428064983
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,2047,0.024890666206677754
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,4095,0.03495466709136963
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,4095,0.035504000882307686
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,8191,0.05446400245030721
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,8191,0.059104000528653465
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,16383,0.09613333145777385
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,16383,0.10188800096511841
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,32767,0.15997866789499918
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,32767,0.1857973337173462
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,65535,0.3001706600189209
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,65535,0.35309334595998126
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,1,0.038506666819254555
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,fp8,1,0.038773333032925926
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,3,0.03830400109291077
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,fp8,3,0.03877866764863332
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,7,0.03866666555404663
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,fp8,7,0.038831998904546104
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,15,0.03817066550254822
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,fp8,15,0.038975998759269714
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,31,0.03827733298142751
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,fp8,31,0.039034667114416756
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,63,0.039317332208156586
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,fp8,63,0.038933334251244865
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,127,0.045279999574025474
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,fp8,127,0.04861866434415182
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,255,0.05718400080998739
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,fp8,255,0.06353599826494853
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,1,0.06469333171844482
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,fp8,1,0.0629120022058487
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,3,0.06433066725730896
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,fp8,3,0.0627040018637975
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,fp8,7,0.06311466793219249
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,7,0.06477866570154826
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,fp8,15,0.06329600016276042
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,15,0.06463466584682465
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,31,0.06749333441257477
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,fp8,31,0.06322666505972545
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,63,0.07133333384990692
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,fp8,63,0.06512000163396199
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,127,0.07417066891988118
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,255,0.09650133053461711
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,fp8,127,0.0888213316599528
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,fp8,255,0.10512000322341919
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,1,0.11569066842397054
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,fp8,1,0.11386666695276897
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,3,0.11533332864443462
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,fp8,3,0.11333333452542622
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,31,0.1260693371295929
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,fp8,31,0.11685333649317424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,7,0.11598400274912517
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,15,0.12110933661460876
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,fp8,15,0.11357866724332173
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,63,0.12758400042851767
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,fp8,7,0.11377066373825073
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,fp8,63,0.12380266189575195
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,fp8,127,0.1588159998257955
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,127,0.13157866398493448
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,fp8,255,0.18641066551208496
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,255,0.17509865760803223
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,1,0.06489066779613495
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,fp8,1,0.06622399886449178
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,fp8,3,0.06598933537801106
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,3,0.06498133142789204
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,7,0.06525866687297821
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,fp8,7,0.0660693347454071
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,31,0.07105599840482076
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,fp8,31,0.06665066878000896
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,15,0.06547733147939046
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,fp8,15,0.06567466755708058
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,63,0.07282133400440216
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,127,0.0759626676638921
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,fp8,63,0.07225599884986877
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,1,0.11633599797884624
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,fp8,127,0.0906933347384135
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,fp8,1,0.11348266402880351
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,fp8,3,0.11362133423487346
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,7,0.11720533172289531
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,fp8,7,0.11391466856002808
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,15,0.12443199753761292
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,fp8,31,0.12088533242543538
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,31,0.1283573309580485
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,3,0.11592533191045125
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,63,0.12823999921480814
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,fp8,63,0.1255466639995575
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,127,0.13337600231170654
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,fp8,127,0.15734933813412985
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,1,0.2193173368771871
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,fp8,1,0.2163626750310262
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,fp8,15,0.11360533038775127
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,3,0.2185493310292562
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,fp8,3,0.21503466367721558
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,7,0.232095996538798
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,fp8,15,0.22499199708302817
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,15,0.23633599281311035
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,fp8,31,0.23110399643580118
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,31,0.2394719918568929
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,1,0.011509332805871964
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,1,0.012170666207869848
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,3,0.012357333054145178
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,63,0.23987199862798056
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,7,0.011407999942700068
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,3,0.011402666568756104
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,7,0.012304000556468964
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,15,0.011343999455372492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,31,0.011637333780527115
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,31,0.012351999680201212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,63,0.011253333340088526
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,15,0.012223999947309494
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,63,0.012256000190973282
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,127,0.011557333171367645
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,127,0.01328533391157786
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,255,0.012671999633312225
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,255,0.014453332871198654
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,511,0.014869333555301031
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,511,0.016986666868130367
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,fp8,63,0.23267199595769247
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,1023,0.01903466631968816
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,1023,0.021669333179791767
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,2047,0.02779199928045273
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,2047,0.0314026673634847
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,4095,0.046069333950678505
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,8191,0.08275733391443889
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,8191,0.09308800101280212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,16383,0.15221866965293884
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,4095,0.050842667619387306
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,fp8,7,0.2151040037473043
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,16383,0.18291199207305908
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,32767,0.2916266719500224
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,32767,0.34971733887990314
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,1,0.01181866725285848
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,1,0.013013333082199097
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,3,0.011514666179815928
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,3,0.012890666723251343
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,7,0.011589333415031433
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,7,0.012944000462690989
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,15,0.01157333329319954
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,127,0.24803733825683594
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,15,0.012965332716703415
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,31,0.011477333803971609
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,31,0.012837332983811697
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,63,0.011701333026091257
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,fp8,127,0.30104533831278485
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,63,0.01267733300725619
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,127,0.011781333635250727
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,127,0.014378666877746582
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,255,0.014805333067973455
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,511,0.015247999380032221
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,511,0.01803733284274737
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,1023,0.01929066702723503
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,2047,0.02834133307139079
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,255,0.013530666629473368
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,4095,0.047557334105173744
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,2047,0.03232000023126602
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,4095,0.05212266743183136
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,8191,0.08323200047016144
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,8191,0.09827733039855957
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,16383,0.15339199701944986
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,1023,0.022704000274340313
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,16383,0.18199467658996582
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,32767,0.2945386568705241
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,1,0.012122667084137598
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,32767,0.35288000106811523
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,1,0.01331199953953425
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,3,0.01192533348997434
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,3,0.01313599944114685
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,7,0.01198400060335795
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,7,0.013471999516089758
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,15,0.012015999605258306
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,15,0.013194666554530462
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,31,0.01191466674208641
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,31,0.013248000293970108
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,63,0.011978667229413986
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,63,0.013455999394257864
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,127,0.012533333152532578
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,127,0.015109332899252573
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,255,0.013605333864688873
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,255,0.015583999454975128
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,511,0.015226667126019796
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,511,0.018277333428462345
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,1023,0.02075200031201045
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,1023,0.0227360005180041
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,2047,0.032458665470282234
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,2047,0.03289066751797994
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,4095,0.05275199810663859
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,4095,0.0572266678015391
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,8191,0.08945066730181377
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,8191,0.09935999910036723
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,16383,0.17154133319854736
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,16383,0.18302400906880698
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,1,0.011941333611806234
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,32767,0.3187413414319356
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,1,0.012597333639860153
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,32767,0.35053332646687824
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,3,0.012282667060693106
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,3,0.012671999633312225
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,31,0.011909333368142446
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,7,0.012080000092585882
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,7,0.013237333546082178
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,15,0.012191999703645706
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,15,0.012629333883523941
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,31,0.012730666746695837
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,63,0.01184533288081487
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,63,0.012703999876976013
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,127,0.012223999947309494
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,127,0.014149333039919535
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,255,0.014021333307027817
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,255,0.01452800010641416
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,1023,0.02183466653029124
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,511,0.015477333217859268
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,2047,0.029109333952267964
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,511,0.017397332936525345
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,1023,0.019760000209013622
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,2047,0.03212266663710276
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,4095,0.04822400212287903
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,4095,0.05211733281612396
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,8191,0.0827466646830241
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,8191,0.09427733222643535
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,16383,0.15296533703804016
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,16383,0.17251733938852945
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,3,0.011717333147923151
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,32767,0.29175466299057007
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,32767,0.3311520020167033
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,1,0.012485332787036896
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,1,0.013642666240533194
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,3,0.013056000073750814
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,7,0.011727999895811081
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,7,0.012773333738247553
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,15,0.012096000214417776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,15,0.013125333935022354
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,31,0.012298667182525
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,31,0.013605333864688873
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,63,0.012058666596810022
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,63,0.012853333105643591
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,127,0.012053333222866058
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,127,0.015029333531856537
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,255,0.013493333011865616
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,1023,0.0206986665725708
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,255,0.015386667102575302
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,511,0.01544533297419548
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,2047,0.03335466732581457
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,511,0.018373332917690277
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,1023,0.02239466706911723
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,2047,0.03262399882078171
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,4095,0.05144000053405762
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,4095,0.056602666775385536
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,8191,0.09384000301361084
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,8191,0.09966400265693665
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,16383,0.17329599459966025
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,32767,0.3529760042826335
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,16383,0.18483734130859375
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,1,0.012261333564917246
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,32767,0.29953600962956745
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,1,0.013754667093356451
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,3,0.012495999534924826
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,3,0.013605333864688873
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,7,0.012330666184425354
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,7,0.013573333621025085
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,15,0.012378666549921036
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,15,0.013466666142145792
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,63,0.013647999614477158
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,31,0.012608000387748083
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,31,0.01368533323208491
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,63,0.012351999680201212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,127,0.012543999900420507
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,127,0.015317333241303762
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,255,0.01431999976436297
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,255,0.015706667055686314
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,511,0.016293333222468693
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,511,0.018538666268189747
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,1023,0.02882133424282074
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,1023,0.02313599983851115
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,2047,0.04404800136884054
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,2047,0.038176000118255615
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,4095,0.07397333284219106
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,4095,0.058789332707722984
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,8191,0.13452800114949545
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,8191,0.10102400183677673
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,16383,0.2534559965133667
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,16383,0.18590933084487915
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,1,0.016607999801635742
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,32767,0.3542506694793701
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,1,0.01756799966096878
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,32767,0.498149315516154
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,3,0.016528000434239704
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,3,0.017727999637524288
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,7,0.016255999604860943
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,7,0.01759999990463257
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,31,0.017818666994571686
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,15,0.017637333522240322
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,15,0.016255999604860943
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,31,0.016415999581416447
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,63,0.016480000068744022
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,63,0.017845333864291508
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,127,0.016927999754746754
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,127,0.02072000006834666
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,255,0.020474666108687718
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,255,0.022085333863894146
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,511,0.030154667794704437
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,511,0.0271573339899381
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,1023,0.0462666650613149
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,1023,0.041402667760849
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,2047,0.07597866654396057
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,2047,0.06284800171852112
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,4095,0.1350826621055603
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,4095,0.10446400443712871
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,8191,0.25282132625579834
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,8191,0.1898933251698812
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,16383,0.49374934037526447
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,16383,0.3549333413441976
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,1,0.011231999844312668
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,1,0.012138667205969492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,3,0.011034666250149408
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,7,0.012005332857370377
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,3,0.012144000579913458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,32767,0.6883573532104492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,7,0.011077333241701126
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,32767,1.0375466346740723
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,15,0.011370666325092316
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,15,0.012250666817029318
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,31,0.011301333705584208
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,31,0.012154666086037954
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,63,0.011152000476916632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,511,0.012298667182525
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,127,0.011178666104873022
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,63,0.012133333832025528
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,127,0.012975999464591345
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,255,0.012250666817029318
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,255,0.013093333691358566
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,2047,0.015061333775520325
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,511,0.013199999928474426
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,1023,0.012576000144084295
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,1023,0.013568000247081121
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,2047,0.013957332819700241
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,4095,0.015237333873907724
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,4095,0.01617066686352094
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,8191,0.01764800027012825
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,8191,0.018863999595244724
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,16383,0.022330666581789654
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,16383,0.024256000916163128
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,32767,0.03194133440653483
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,32767,0.03427733232577642
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,65535,0.05036266644795736
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,65535,0.055829331278800964
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,131071,0.08904000123341878
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,131071,0.09781333804130554
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,1,0.011055999745925268
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,1,0.012186666329701742
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,3,0.011029332876205444
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,3,0.012159999459981918
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,7,0.011087999989589056
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,7,0.01251199965675672
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,15,0.011359999577204386
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,15,0.012138667205969492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,31,0.011157333850860596
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,31,0.012282667060693106
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,63,0.01091733326514562
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,63,0.012357333054145178
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,127,0.011194666226704916
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,127,0.01312000056107839
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,255,0.012213333199421564
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,255,0.013237333546082178
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,511,0.012362666428089142
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,511,0.013610667238632837
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,1023,0.012858666479587555
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,1023,0.014629332969586054
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,2047,0.01421333352724711
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,2047,0.01591466615597407
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,4095,0.015930666277805965
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,4095,0.017242666333913803
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,8191,0.018320000420014065
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,8191,0.019952000429232914
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,16383,0.022810667753219604
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,16383,0.02532266577084859
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,32767,0.03324799984693527
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,32767,0.035402665535608925
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,65535,0.05390933156013489
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,65535,0.05693866809209188
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,131071,0.08988266189893086
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,131071,0.10790933171908061
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,1,0.011354666203260422
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,1,0.012144000579913458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,3,0.011391999820868174
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,3,0.012074666718641916
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,7,0.011215999722480774
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,7,0.012293333808581034
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,15,0.011717333147923151
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,15,0.012256000190973282
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,31,0.011152000476916632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,31,0.012229333321253458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,63,0.011381333072980246
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,63,0.01228800043463707
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,127,0.013232000172138214
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,127,0.011424000064531961
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,255,0.012506666282812754
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,255,0.013445333888133367
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,511,0.012725333372751871
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,511,0.014736000448465347
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,4095,0.016821333517630894
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,1023,0.013621332744757334
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,1023,0.01509333277742068
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,2047,0.01524266724785169
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,2047,0.01648533344268799
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,4095,0.01732800031701724
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,8191,0.018735999862353008
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,8191,0.02006400004029274
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,16383,0.02386666586001714
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,16383,0.025631998976071674
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,32767,0.035962666074434914
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,32767,0.036474667489528656
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,65535,0.057162667314211525
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,65535,0.061717331409454346
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,131071,0.0940106709798177
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,131071,0.10830400387446086
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,1,0.011472000430027643
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,1,0.012053333222866058
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,3,0.011007999380429586
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,3,0.012256000190973282
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,7,0.011920000116030375
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,7,0.01240533341964086
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,15,0.01121066634853681
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,15,0.012362666428089142
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,31,0.011578666667143503
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,31,0.012229333321253458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,63,0.011407999942700068
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,63,0.012229333321253458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,127,0.011503999431928
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,127,0.013424000392357508
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,255,0.012645332763592402
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,255,0.014549333602190018
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,511,0.013642666240533194
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,511,0.014677333335081736
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,1023,0.014032000054915747
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,1023,0.01532799998919169
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,2047,0.015893333901961643
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,2047,0.017152000218629837
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,4095,0.0170666662355264
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,4095,0.017973333597183228
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,8191,0.01912533367673556
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,8191,0.020725333442290623
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,16383,0.030026666820049286
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,16383,0.026074667771657307
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,32767,0.046709333856900535
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,32767,0.04065600037574768
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,65535,0.07876800000667572
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,65535,0.06334400177001953
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,131071,0.14404799540837607
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,131071,0.10890133182207744
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,1,0.010992000500361124
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,1,0.012047999848922094
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,3,0.011391999820868174
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,3,0.012346666306257248
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,7,0.011071999867757162
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,7,0.011887999872366587
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,15,0.011205332974592844
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,15,0.01191466674208641
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,31,0.011242666592200598
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,31,0.01201066623131434
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,63,0.011333333949247995
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,63,0.012085333466529846
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,127,0.011120000233252844
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,127,0.013370666652917862
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,255,0.01219733307758967
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,255,0.012869333227475485
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,511,0.012432000289360682
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,511,0.013354666531085968
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,1023,0.013631999492645264
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,1023,0.0145066666106383
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,2047,0.014997333288192749
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,2047,0.015573333948850632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,4095,0.01754133279124896
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,4095,0.018394666413466137
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,8191,0.0206133338312308
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,8191,0.022837333381175995
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,16383,0.029098667204380035
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,16383,0.03323733309904734
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,32767,0.04655466477076212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,131071,0.15126400192578635
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,32767,0.05240533252557119
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,65535,0.08257600168387096
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,65535,0.09232532978057861
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,131071,0.17314134041468301
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,1,0.011338666081428528
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,1,0.011989332735538483
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,3,0.011007999380429586
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,3,0.012170666207869848
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,7,0.01110400011142095
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,7,0.012351999680201212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,15,0.011141333729028702
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,15,0.012543999900420507
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,31,0.01109333336353302
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,31,0.012240000069141388
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,63,0.01118933285276095
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,63,0.012170666207869848
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,127,0.013386666774749756
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,127,0.011328000575304031
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,255,0.012362666428089142
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,255,0.013738666971524557
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,511,0.01267733300725619
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,511,0.014368000129858652
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,1023,0.013679999858140945
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,1023,0.015562667200962702
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,2047,0.015685333559910457
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,2047,0.016735999534527462
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,4095,0.0179626668492953
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,4095,0.019413333386182785
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,8191,0.02143999934196472
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,8191,0.023775999744733173
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,16383,0.030117332935333252
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,16383,0.03385066737731298
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,32767,0.04896533489227295
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,32767,0.05342933535575867
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,65535,0.08444799979527791
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,65535,0.10122666756312053
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,131071,0.15593600273132324
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,1,0.011285333583752314
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,131071,0.1830880045890808
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,1,0.012106666962305704
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,3,0.011519999553759893
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,3,0.012319999436537424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,7,0.011381333072980246
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,7,0.012560000022252401
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,15,0.011343999455372492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,15,0.012144000579913458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,31,0.011424000064531961
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,31,0.01227733368674914
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,63,0.011525332927703857
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,63,0.012527999778588613
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,127,0.011493333925803503
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,127,0.01358933374285698
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,255,0.012495999534924826
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,255,0.014229333649079004
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,511,0.013280000537633896
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,511,0.014959999670584997
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,1023,0.014970666418472925
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,1023,0.016202667107184727
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,2047,0.016229332735141117
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,2047,0.016906666258970898
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,4095,0.018378666291634243
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,4095,0.01972266659140587
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,8191,0.02181333303451538
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,8191,0.024351999163627625
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,16383,0.03843733419974645
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,16383,0.035029334326585136
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,32767,0.054383998115857445
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,32767,0.058778668443361916
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,65535,0.11241599917411804
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,65535,0.10089066624641418
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,131071,0.17567465702692667
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,131071,0.18382400274276733
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,1,0.0116799995303154
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,1,0.012533333152532578
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,3,0.011589333415031433
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,3,0.012543999900420507
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,7,0.01157333329319954
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,7,0.012437333663304647
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,15,0.011834666132926941
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,15,0.012576000144084295
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,31,0.011786667009194693
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,31,0.012730666746695837
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,63,0.011706666400035223
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,63,0.012560000022252401
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,127,0.011578666667143503
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,127,0.013951999445756277
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,255,0.013471999516089758
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,255,0.014922666052977243
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,511,0.013994666437307993
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,511,0.015247999380032221
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,1023,0.015189333508412043
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,1023,0.016341333587964375
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,4095,0.020106667031844456
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,2047,0.017322666943073273
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,2047,0.01664000004529953
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,4095,0.01870399961868922
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,8191,0.03052799900372823
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,8191,0.025077333052953083
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,16383,0.045007998744646706
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,16383,0.03908800085385641
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,32767,0.06332799792289734
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,32767,0.07609066863854726
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,65535,0.13552000125249228
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,1,0.012773333738247553
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,65535,0.10717866818110149
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,131071,0.18736000855763754
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,131071,0.2590133349100749
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,1,0.013253333667914072
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,3,0.012826666235923767
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,3,0.01370666672786077
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,7,0.01358933374285698
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,7,0.013199999928474426
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,15,0.013104000439246496
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,15,0.013306666165590286
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,31,0.013248000293970108
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,31,0.013327999661366144
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,63,0.013167999684810638
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,63,0.013264000415802002
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,127,0.013125333935022354
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,127,0.014639999717473984
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,511,0.018330667167901993
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,255,0.014906667172908783
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,255,0.01543466622630755
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,511,0.01618133361140887
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,1023,0.022015998760859173
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,1023,0.02271466702222824
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,2047,0.03414933383464813
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,2047,0.03365333378314972
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,8191,0.09530133008956909
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,4095,0.04974400003751119
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,8191,0.09504533807436626
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,1,0.012901333471139273
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,4095,0.055493334929148354
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,16383,0.17133333285649618
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,16383,0.1727466583251953
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,1,0.013765333841244379
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,3,0.012954667210578918
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,3,0.01379199946920077
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,7,0.013232000172138214
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,7,0.013765333841244379
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,15,0.013354666531085968
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,15,0.013978666315476099
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,31,0.01314666618903478
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,31,0.013957332819700241
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,63,0.013007999708255133
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,63,0.013989333063364029
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,127,0.013424000392357508
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,127,0.015173333386580149
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,255,0.015103999525308609
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,255,0.016208000481128693
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,511,0.01736533393462499
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,511,0.018778666853904724
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,1023,0.02890666574239731
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,1023,0.023631999890009563
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,2047,0.04463466505209605
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,2047,0.03835733234882355
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,4095,0.07625600198904674
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,8191,0.10256000359853108
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,4095,0.05963733295599619
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,8191,0.135343998670578
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,16383,0.2553333242734273
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,16383,0.18725866079330444
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,1,0.01695466662446658
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,3,0.01729600007335345
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,1,0.017978666971127193
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,3,0.018650667121013004
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,7,0.01716800034046173
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,31,0.017242666333913803
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,7,0.017797333498795826
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,63,0.017029333859682083
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,15,0.01703466723362605
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,15,0.017978666971127193
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,31,0.01829333355029424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,63,0.018357332795858383
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,127,0.017551999539136887
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,127,0.021253332495689392
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,255,0.02073066681623459
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,255,0.022240000466505688
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,511,0.030917334059874218
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,511,0.027866666515668232
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,1023,0.04649066428343455
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,1023,0.04177600145339966
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,2047,0.0772159993648529
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,2047,0.06367999811967213
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,4095,0.13492266337076822
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,4095,0.1053493320941925
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,8191,0.25285865863164264
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,8191,0.19050133228302002
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,1,0.023813332120577495
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,1,0.025493333737055462
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,3,0.023887999355793
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,16383,0.35735468069712323
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,16383,0.487498680750529
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,3,0.02532266577084859
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,7,0.023919999599456787
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,7,0.025407999753952026
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,15,0.023872000475724537
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,15,0.02513066679239273
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,31,0.024266667664051056
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,31,0.025781333446502686
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,63,0.024069334069887798
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,63,0.02593066543340683
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,127,0.02500266581773758
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,127,0.030095999439557392
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,255,0.03691199918588003
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,255,0.03402666747570038
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,1023,0.08101333181063335
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,2047,0.13799466689427695
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,511,0.05054399867852529
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,511,0.04933333396911621
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,1023,0.06950399776299794
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,2047,0.11270399888356526
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,4095,0.254746675491333
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,4095,0.19497066736221313
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,8191,0.3615146478017171
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,1,0.011221333096424738
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,8191,0.48868799209594727
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,1,0.011354666203260422
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,3,0.010762666662534079
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,3,0.011477333803971609
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,7,0.010768000036478043
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,31,0.010805333654085795
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,16383,1.0408960183461506
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,7,0.012074666718641916
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,16383,0.7006613413492838
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,15,0.010890666395425797
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,15,0.011418666690587997
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,31,0.011541333049535751
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,63,0.0107893335322539
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,63,0.011648000528415045
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,255,0.012165332833925882
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,127,0.013056000073750814
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,127,0.010746666540702185
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,255,0.012901333471139273
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,511,0.013023999830087027
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,511,0.013909333695967993
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,1023,0.014256000518798828
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,1023,0.015114666273196539
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,2047,0.015578666081031164
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,2047,0.017935999979575474
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,4095,0.020037333170572918
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,4095,0.02236266682545344
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,8191,0.02864533414443334
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,32767,0.09330133597056071
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,8191,0.03270400067170461
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,16383,0.04653333127498627
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,16383,0.052000001072883606
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,131071,0.29077333211898804
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,32767,0.08322133123874664
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,65535,0.1516480048497518
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,65535,0.1723466714223226
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,131071,0.3307680090268453
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,1,0.01097600037852923
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,1,0.01181866725285848
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,3,0.010949333508809408
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,3,0.011653333902359009
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,7,0.011039999624093374
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,7,0.011882666498422623
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,15,0.010890666395425797
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,127,0.01110400011142095
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,15,0.01231466606259346
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,31,0.01101333275437355
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,31,0.011733333269755045
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,63,0.011002667248249054
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,63,0.011744000017642975
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,511,0.015173333386580149
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,1023,0.015360000232855478
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,127,0.013167999684810638
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,255,0.012149333953857422
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,255,0.013882666826248169
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,511,0.013450667262077332
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,1023,0.01597333326935768
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,2047,0.01642666632930438
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,2047,0.01871466636657715
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,4095,0.02057066683967908
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,4095,0.023200000325838726
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,32767,0.08426133791605632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,8191,0.030192000170548756
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,8191,0.03348266581694285
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,65535,0.1534346640110016
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,16383,0.04919999837875366
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,16383,0.053226664662361145
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,131071,0.2954453428586324
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,32767,0.10005333026250203
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,65535,0.18428266048431396
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,1,0.011461333682139715
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,1,0.012416000167528788
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,131071,0.3530079921086629
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,3,0.011359999577204386
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,3,0.012362666428089142
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,7,0.011343999455372492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,7,0.012117333710193634
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,15,0.011152000476916632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,15,0.012426666915416718
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,31,0.01145600030819575
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,31,0.012383999923865
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,63,0.01138666644692421
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,63,0.012432000289360682
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,127,0.011194666226704916
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,127,0.013744000345468521
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,255,0.013301332791646322
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,1023,0.015834666788578033
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,255,0.0141546664138635
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,511,0.014853333433469137
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,511,0.015658666690190632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,1023,0.01664000004529953
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,2047,0.01651200031240781
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,2047,0.018906666586796444
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,4095,0.021695998807748158
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,4095,0.023786666492621105
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,8191,0.03384533276160558
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,8191,0.034101332227389015
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,16383,0.05401599903901418
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,16383,0.05788266658782959
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,32767,0.09296533465385437
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,32767,0.09986133376757304
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,65535,0.1746079921722412
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,65535,0.18386665980021158
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,1,0.011717333147923151
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,131071,0.39976000785827637
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,131071,0.3535733222961426
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,1,0.013066666821638743
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,3,0.011616000284751257
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,3,0.01293333371480306
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,7,0.011727999895811081
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,7,0.01267733300725619
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,15,0.011589333415031433
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,15,0.013178666432698568
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,31,0.011973333855470022
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,31,0.013162666310866674
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,63,0.011642667154471079
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,63,0.012831999609867731
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,127,0.011994666109482447
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,127,0.0143306665122509
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,255,0.013088000317414602
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,255,0.014773332824309668
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,511,0.014746667196353277
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,511,0.015978666643301647
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,1023,0.015482666591803232
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,1023,0.016832000265518825
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,2047,0.01736533393462499
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,2047,0.019296000401178997
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,4095,0.02993600070476532
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,4095,0.024277334411938984
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,8191,0.045168002446492515
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,8191,0.03825599948565165
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,16383,0.07618133227030437
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,16383,0.05925333499908447
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,32767,0.13807466626167297
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,32767,0.10155199964841206
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,65535,0.25810132424036664
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,65535,0.18582934141159058
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,131071,0.5215839942296346
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,131071,0.355461319287618
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,1,0.014346666634082794
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,fp8,1,0.015135999768972397
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,fp8,3,0.015279999623696009
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,3,0.014159999787807465
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,7,0.014122666170199713
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,fp8,7,0.015077333897352219
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,15,0.013978666315476099
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,fp8,15,0.015119999647140503
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,31,0.014378666877746582
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,fp8,31,0.015354666858911514
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,63,0.014522666732470194
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,fp8,63,0.015018666783968607
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,127,0.014853333433469137
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,fp8,255,0.017136000096797943
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,fp8,127,0.016800000021855038
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,fp8,511,0.019925333559513092
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,255,0.01613866661985715
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,511,0.018330667167901993
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,1023,0.03046400099992752
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,fp8,1023,0.025253333151340485
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,2047,0.046096002062161766
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,fp8,2047,0.037808001041412354
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,8191,0.13738133509953818
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,fp8,8191,0.09949333469072978
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,1,0.017605333278576534
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,fp8,4095,0.05859733124574026
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,4095,0.07751466830571492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,fp8,1,0.018709332992633183
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,3,0.0176959993938605
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,fp8,3,0.018944000204404194
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,7,0.017711999515692394
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,fp8,7,0.01883200059334437
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,31,0.017722666263580322
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,fp8,31,0.01894933357834816
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,15,0.017605333278576534
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,63,0.017653333644072216
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,fp8,63,0.019007999449968338
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,fp8,15,0.019007999449968338
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,127,0.017978666971127193
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,fp8,127,0.021477334201335907
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,255,0.021909333765506744
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,fp8,255,0.023311999936898548
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,511,0.0330826664964358
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,fp8,511,0.02867199977238973
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,1023,0.048810665806134544
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,fp8,1023,0.042992000778516136
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,2047,0.07820799946784973
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,4095,0.13756266236305237
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,fp8,4095,0.10719466209411621
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,8191,0.25447465976079303
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,fp8,2047,0.06515199939409892
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,1,0.024405332903067272
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,fp8,8191,0.1925706664721171
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,3,0.024442667762438457
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,fp8,3,0.026165333886941273
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,7,0.024506665766239166
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,fp8,1,0.026208000878492992
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,15,0.024506665766239166
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,fp8,15,0.02624000112215678
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,31,0.024725332856178284
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,fp8,31,0.02638400097688039
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,fp8,7,0.0259253333012263
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,63,0.02474133421977361
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,fp8,63,0.026752000053723652
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,127,0.02532800038655599
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,fp8,127,0.030794667700926464
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,255,0.03755199909210205
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,fp8,255,0.03505599995454153
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,511,0.05102399984995524
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,fp8,511,0.050613333781560264
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,2047,0.13784533739089966
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,fp8,2047,0.11262399951616923
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,1023,0.08198399841785431
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,fp8,1023,0.07100266714890797
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,4095,0.2550080021222432
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,fp8,4095,0.19637332359949747
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,1,0.03760000069936117
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,fp8,1,0.040106666584809623
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,8191,0.4898933172225952
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,7,0.03741333385308584
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,fp8,7,0.040218666195869446
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,15,0.03772799919048945
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,fp8,15,0.04026666780312856
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,31,0.03799466788768768
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,fp8,8191,0.3654400110244751
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,fp8,31,0.040405333042144775
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,fp8,63,0.04045866678158442
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,3,0.037589333951473236
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,63,0.038149334490299225
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,fp8,3,0.0406986673672994
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,127,0.04457066456476847
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,fp8,127,0.04993066688378652
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,fp8,1023,0.12430933117866516
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,1023,0.1450506647427877
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,fp8,255,0.0614879975716273
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,fp8,4095,0.37347733974456787
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,4095,0.49264001846313477
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,fp8,2047,0.21050665775934854
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,2047,0.25910399357477826
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,fp8,8191,0.7064053217569987
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,255,0.0572266678015391
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,1,0.019589333484570186
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,fp8,1,0.020784000555674236
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,511,0.08805333574612935
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,3,0.019925333559513092
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,fp8,511,0.08338133494059245
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,fp8,3,0.02086399992307027
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,15,0.019952000429232914
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,fp8,7,0.021221332252025604
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,7,0.01982933282852173
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,31,0.020143999407688778
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,63,0.02004266654451688
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,fp8,63,0.021018666525681812
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,8191,0.9572160243988037
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,127,0.02067733307679494
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,fp8,127,0.023520000278949738
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,fp8,15,0.021087999145189922
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,255,0.024698667228221893
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,fp8,31,0.02109333376089732
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,fp8,255,0.025797332326571148
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,511,0.03569599986076355
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,1023,0.0518453319867452
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,fp8,1023,0.045109331607818604
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,1,0.02611200014750163
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,fp8,511,0.03155199935038885
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,2047,0.08182933429876964
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,fp8,2047,0.06570133566856384
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,fp8,1,0.02804800122976303
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,7,0.026165333886941273
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,3,0.026378666361172993
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,fp8,3,0.02773866554101308
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,fp8,7,0.027823999524116516
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,31,0.026357332865397137
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,15,0.026352000733216602
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,fp8,15,0.02788266787926356
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,fp8,31,0.02775999903678894
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,63,0.026816000541051228
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,fp8,63,0.027935999135176342
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,127,0.026943999032179516
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,fp8,127,0.03265066693226496
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,255,0.040762667854626976
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,fp8,255,0.03649600098530451
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,511,0.05524266759554545
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,fp8,511,0.053264002005259194
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,1023,0.08482133348782857
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,1,0.039221333960692085
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,fp8,1023,0.0737066666285197
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,fp8,1,0.04223466912905375
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,3,0.03945599993069967
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,fp8,2047,0.1167626678943634
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,2047,0.14283733566602072
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,7,0.039162665605545044
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,fp8,3,0.042303999265034996
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,fp8,15,0.04227200150489807
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,15,0.0395359992980957
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,fp8,7,0.0422986646493276
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,31,0.03950933367013931
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,fp8,63,0.04199466605981191
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,127,0.045994664231936135
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,fp8,127,0.0509493350982666
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,255,0.06205866734186808
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,fp8,255,0.06391466657320659
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,511,0.09032533566157024
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,fp8,511,0.08638933300971985
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,fp8,31,0.04207466542720795
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,2047,0.26150399446487427
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,63,0.03993066648642222
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,fp8,2047,0.2135466734568278
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,3,0.06508799890677135
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,1023,0.14597333470980325
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,fp8,3,0.06913066903750102
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,7,0.0653599997361501
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,fp8,1023,0.12636267145474753
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,fp8,7,0.06951466699441274
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,1,0.065461332599322
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,fp8,1,0.06900799771149953
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,15,0.0656160016854604
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,31,0.06584000090758006
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,fp8,31,0.06977066894372304
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,fp8,63,0.07042133311430614
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,127,0.07361066838105519
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,fp8,127,0.09118933478991191
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,255,0.10141332944234212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,fp8,1023,0.22972800334294638
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,fp8,255,0.10609066486358643
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,1023,0.2687679926554362
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,fp8,15,0.06931733091672261
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,63,0.07054399947325389
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,fp8,2047,0.40271464983622235
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,2047,0.49599464734395343
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,fp8,511,0.1509173313776652
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,1,0.03322133421897888
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,fp8,1,0.034789333740870156
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,511,0.15825066963831583
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,3,0.033071999748547874
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,fp8,3,0.034645333886146545
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,7,0.033088001112143196
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,fp8,7,0.03440000116825104
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,fp8,15,0.034688000877698265
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,31,0.033610666791598
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,63,0.03293866664171219
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,15,0.03345600018898646
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,fp8,63,0.03472000112136205
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,127,0.03435733417669932
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,fp8,127,0.03844266633192698
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,fp8,31,0.034346667428811394
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,255,0.04754666487375895
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,fp8,255,0.04548266530036926
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,511,0.06366399923960368
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,1,0.04510400195916494
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,fp8,511,0.060821334520975746
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,1023,0.09191466371218364
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,fp8,1023,0.08010666569073994
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,fp8,1,0.047695999344189964
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,3,0.0451200008392334
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,fp8,3,0.04827733337879181
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,7,0.04524266719818115
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,fp8,7,0.04714666803677877
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,15,0.04535466432571411
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,fp8,31,0.047930667797724404
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,63,0.04604800045490265
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,fp8,63,0.04830400149027506
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,fp8,15,0.04763199885686239
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,31,0.045791998505592346
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,127,0.05444799860318502
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,fp8,127,0.05659733215967814
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,255,0.07106133302052815
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,fp8,255,0.07230933507283528
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,511,0.09887466828028361
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,1,0.07207466661930084
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,fp8,511,0.09521599610646565
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,fp8,1,0.07688533266385396
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,3,0.0718453327814738
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,fp8,1023,0.13615999619166055
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,1023,0.15476266543070474
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,7,0.0718399981657664
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,fp8,3,0.07745066781838734
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,15,0.07220800220966339
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,fp8,7,0.07656000057856242
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,fp8,15,0.07635200023651123
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,63,0.07891199986139934
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,fp8,63,0.07720533510049184
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,127,0.0825973351796468
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,fp8,127,0.09949866930643718
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,255,0.10693333546320598
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,fp8,255,0.11627733707427979
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,31,0.07433600227038066
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,fp8,31,0.07660266757011414
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,3,0.12385599811871846
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,511,0.1644213298956553
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,fp8,1,0.12982400258382162
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,fp8,511,0.1621386706829071
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,fp8,3,0.12990933656692505
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,1,0.12386133273442586
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,7,0.1239413321018219
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,15,0.124208003282547
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,1023,0.2746773362159729
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,fp8,1023,0.24226667483647665
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,31,0.12982933719952902
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,fp8,31,0.13270933429400125
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,63,0.13194132844607034
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,fp8,63,0.1368160049120585
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,fp8,7,0.13110933701197305
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,fp8,15,0.13034133116404215
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,fp8,1023,0.4438399871190389
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,1,0.011141333729028702
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,1,0.011765333513418833
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,255,0.18506133556365967
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,3,0.011359999577204386
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,fp8,255,0.19833066066106161
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,1023,0.5179839928944906
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,127,0.13726933797200522
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,7,0.011087999989589056
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,31,0.011349332829316458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,31,0.011839999506870905
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,63,0.011312000453472137
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,15,0.011130666981140772
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,63,0.011962667107582092
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,127,0.011152000476916632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,15,0.0122079998254776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,127,0.013162666310866674
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,255,0.012234666695197424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,fp8,127,0.1706506609916687
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,255,0.014127999544143677
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,511,0.01357866699496905
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,511,0.015135999768972397
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,1023,0.01758933315674464
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,2047,0.020143999407688778
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,fp8,511,0.2852960030237834
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,2047,0.022330666581789654
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,4095,0.028837333122889202
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,3,0.011909333368142446
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,8191,0.05202133456865946
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,4095,0.032416000962257385
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,32767,0.15253333250681558
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,16383,0.09096533060073853
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,32767,0.17241599162419638
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,16383,0.083146666487058
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,7,0.011829332758982977
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,65535,0.29121599594751996
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,65535,0.32787734270095825
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,1,0.011205332974592844
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,131071,0.5682186683019003
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,131071,0.6468693415323893
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,1,0.01201066623131434
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,1023,0.01586666703224182
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,3,0.011370666325092316
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,3,0.012362666428089142
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,7,0.011231999844312668
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,511,0.3071626623471578
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,7,0.012213333199421564
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,15,0.011306667079528173
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,15,0.012416000167528788
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,31,0.011343999455372492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,8191,0.046336000164349876
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,31,0.012159999459981918
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,63,0.011333333949247995
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,127,0.011610666910807291
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,127,0.013381333400805792
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,255,0.01259200026591619
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,255,0.015216000378131866
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,511,0.014309333016475042
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,511,0.016197333733240765
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,1023,0.0161013330022494
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,1023,0.018874666343132656
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,63,0.012053333222866058
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,2047,0.020560000091791153
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,2047,0.023242667317390442
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,4095,0.030400000512599945
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,4095,0.03306666761636734
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,8191,0.04906666775544485
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,8191,0.053530668218930565
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,16383,0.08405333757400513
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,16383,0.1006773312886556
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,32767,0.154341330130895
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,32767,0.18263467152913412
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,65535,0.2943840026855469
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,65535,0.3529119888941447
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,1,0.011717333147923151
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,131071,0.5763946771621704
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,1,0.01302933320403099
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,7,0.013093333691358566
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,131071,0.6873813470204672
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,3,0.011477333803971609
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,3,0.012997332960367203
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,7,0.01145600030819575
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,31,0.01310933381319046
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,63,0.011770666887362799
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,15,0.011866666376590729
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,15,0.013189333180586496
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,31,0.011920000116030375
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,63,0.013125333935022354
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,127,0.011616000284751257
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,127,0.014005333185195923
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,255,0.013354666531085968
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,255,0.016127999871969223
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,511,0.014848000059525171
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,511,0.01617066686352094
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,1023,0.016234666109085083
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,1023,0.018789333601792652
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,2047,0.021168000996112823
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,2047,0.023631999890009563
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,4095,0.03573333223660787
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,4095,0.03385066737731298
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,8191,0.05390933156013489
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,8191,0.05788800120353699
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,16383,0.09659199913342793
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,16383,0.10036266843477885
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,32767,0.17517866690953574
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,32767,0.18339733282725015
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,65535,0.3011253277460734
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,65535,0.3534453312555949
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,1,0.01180800050497055
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,1,0.013104000439246496
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,7,0.012170666207869848
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,131071,0.6539786656697592
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,3,0.011855999628702799
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,131071,0.6871786912282308
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,3,0.013381333400805792
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,7,0.013002666334311167
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,15,0.011781333635250727
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,15,0.01322666679819425
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,31,0.01231466606259346
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,127,0.014597332725922266
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,255,0.013466666142145792
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,31,0.01303999995191892
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,63,0.011717333147923151
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,63,0.013317332913478216
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,127,0.012309333930412928
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,255,0.01611199975013733
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,511,0.015685333559910457
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,511,0.017024000485738117
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,1023,0.016949333250522614
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,1023,0.018885333091020584
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,2047,0.029477333029111225
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,2047,0.024271999796231587
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,4095,0.04472533365090688
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,4095,0.03856533269087473
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,8191,0.0763733337322871
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,8191,0.060005332032839455
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,16383,0.13607466220855713
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,16383,0.10156266887982686
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,32767,0.25779199600219727
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,32767,0.18587199846903482
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,65535,0.5210506518681844
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,65535,0.3543253342310588
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,131071,0.6933066844940186
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,131071,1.0388106505076091
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,1,0.03745066622893015
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,fp8,1,0.037861332297325134
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,3,0.03739733248949051
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,fp8,3,0.037952000896135964
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,fp8,7,0.03792533278465271
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,7,0.03751466671625773
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,15,0.03751466671625773
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,fp8,15,0.03839999934037527
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,31,0.03726933399836222
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,fp8,31,0.03771200031042099
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,63,0.037471999724706016
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,fp8,63,0.03794133414824804
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,127,0.04404800136884054
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,fp8,127,0.04645866652329763
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,255,0.056501333912213646
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,fp8,255,0.062352001667022705
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,1,0.06434666613737743
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,511,0.08577066659927368
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,fp8,511,0.0832426647345225
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,fp8,1,0.06291733185450236
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,3,0.06417599817117055
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,fp8,3,0.0632479985555013
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,7,0.06460266808668773
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,fp8,7,0.0631039987007777
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,31,0.06634133557478587
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,15,0.06447466711203258
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,fp8,31,0.06300800045331319
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,fp8,15,0.06272000074386597
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,63,0.07088533540566762
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,fp8,63,0.06471466521422069
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,127,0.07417599856853485
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,fp8,127,0.08754666646321614
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,255,0.0958666702111562
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,1,0.11517332990964253
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,fp8,255,0.10573333501815796
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,fp8,1,0.11370133360226949
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,511,0.15123732884724936
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,fp8,511,0.15133866667747498
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,3,0.11691733201344807
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,fp8,3,0.11376532912254333
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,fp8,7,0.11314133803049724
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,7,0.11588266491889954
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,fp8,15,0.11355200409889221
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,15,0.11912000179290771
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,63,0.12782933314641318
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,fp8,63,0.12359467148780823
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,31,0.12589866916338602
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,fp8,31,0.1151039997736613
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,127,0.13115732868512472
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,fp8,127,0.15773866573969522
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,1,0.2126026749610901
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,255,0.17353065808614096
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,fp8,1,0.20565332969029745
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,fp8,255,0.1866719921429952
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,3,0.21082133054733276
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,fp8,3,0.20693333943684897
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,fp8,511,0.28036266565322876
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,511,0.2805226643880208
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,7,0.21746132771174112
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,15,0.22614934047063193
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,fp8,7,0.20687466859817505
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,31,0.22688533862431845
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,fp8,31,0.22066134214401245
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,fp8,15,0.21197867393493652
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,127,0.23866132895151773
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,fp8,127,0.2913600007692973
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,63,0.2302293380101522
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,fp8,63,0.22311466932296753
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,1,0.06284800171852112
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,fp8,255,0.3432106574376424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,255,0.3237066666285197
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,fp8,1,0.06322133541107178
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,3,0.0628959983587265
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,fp8,3,0.06445333361625671
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,7,0.06281599899133046
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,511,0.6102826595306396
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,fp8,511,0.5202399889628092
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,fp8,15,0.063360000650088
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,31,0.06398933132489522
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,fp8,31,0.06451733410358429
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,fp8,7,0.06333866715431213
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,15,0.06303466856479645
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,63,0.06971733272075653
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,fp8,127,0.08523733417193095
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,255,0.09602133433024089
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,fp8,255,0.1088266670703888
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,1,0.11574932932853699
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,fp8,1,0.11381866534550984
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,3,0.1165066659450531
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,127,0.07346666852633159
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,fp8,63,0.06719466547171275
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,fp8,3,0.1129973332087199
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,7,0.11729600032170613
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,31,0.12681066989898682
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,15,0.12141333023707072
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,fp8,15,0.11404266953468323
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,fp8,7,0.11463466286659241
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,63,0.12744533022244772
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,fp8,31,0.11782399813334148
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,fp8,63,0.1249066690603892
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,1,0.22082134087880453
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,fp8,1,0.21451199054718018
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,127,0.13209066788355509
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,3,0.219866673151652
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,fp8,127,0.15781333049138388
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,255,0.17563732465108237
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,fp8,255,0.19181867440541586
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,fp8,3,0.21347200870513916
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,7,0.2318026622136434
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,fp8,7,0.21459199984868368
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,fp8,15,0.22215465704600015
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,15,0.23693333069483438
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,31,0.23828800519307455
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,fp8,63,0.23089599609375
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,63,0.23905066649119058
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,fp8,31,0.2286400000254313
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,127,0.2490239938100179
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,fp8,127,0.29925866921742755
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,1,0.4097280104955037
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,255,0.33109867572784424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,fp8,255,0.35631998380025226
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,fp8,1,0.4002240101496379
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,3,0.42123734951019287
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,fp8,3,0.4009439945220947
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,7,0.4434453248977661
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,fp8,7,0.41302935282389325
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,15,0.4397653341293335
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,fp8,31,0.4259573221206665
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,63,0.44418664773305255
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,fp8,63,0.4291199843088786
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,fp8,15,0.42501866817474365
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,31,0.44378666083017987
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,1,0.011813333878914515
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,1,0.011429333438475927
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,3,0.011306667079528173
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,127,0.46511999766031903
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,7,0.012080000092585882
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,3,0.011999999483426413
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,15,0.011328000575304031
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,31,0.011242666592200598
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,31,0.012080000092585882
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,63,0.011482667177915573
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,63,0.012026666353146235
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,127,0.011285333583752314
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,127,0.013125333935022354
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,255,0.012655999511480331
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,255,0.013957332819700241
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,511,0.014965333044528961
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,511,0.016858667135238647
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,1023,0.0189280000825723
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,1023,0.021429332594076794
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,7,0.01139733319481214
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,2047,0.02775466690460841
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,2047,0.031680000325044
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,4095,0.04586133360862732
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,8191,0.08101866642634074
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,4095,0.05136533578236898
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,8191,0.09156266848246257
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,16383,0.17123732964197794
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,16383,0.15119999647140503
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,32767,0.33084799846013385
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,32767,0.2904053330421448
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,65535,0.5633493264516195
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,1,0.01173866664369901
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,1,0.012736000120639801
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,65535,0.645520011583964
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,3,0.011685332904259363
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,3,0.012629333883523941
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,7,0.011621333658695221
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,7,0.012928000340859095
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,15,0.011653333902359009
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,15,0.012682666381200155
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,31,0.01157333329319954
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,fp8,127,0.5650879939397176
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,31,0.012874666601419449
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,63,0.011839999506870905
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,63,0.013045333325862885
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,127,0.011994666109482447
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,127,0.014661333213249842
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,255,0.01498666654030482
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,511,0.014922666052977243
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,511,0.01775466650724411
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,1023,0.019386666516462963
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,1023,0.02258666604757309
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,2047,0.028234665592511494
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,2047,0.0323840007185936
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,4095,0.05268266797065735
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,4095,0.04798933366934458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,15,0.012282667060693106
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,8191,0.08285333216190338
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,16383,0.1532693306605021
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,8191,0.09920000036557515
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,16383,0.18238933881123862
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,32767,0.2945866584777832
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,32767,0.3518720070521037
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,1,0.012063999970753988
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,65535,0.5729493300120035
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,1,0.013584000368913015
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,3,0.01190399999419848
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,65535,0.6875306765238444
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,3,0.013445333888133367
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,7,0.012144000579913458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,7,0.013306666165590286
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,15,0.012047999848922094
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,15,0.013359999905029932
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,31,0.01219733307758967
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,31,0.013418667018413544
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,63,0.011999999483426413
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,63,0.013248000293970108
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,127,0.01209066684047381
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,127,0.014522666732470194
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,255,0.01360000049074491
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,255,0.013610667238632837
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,255,0.015775999675194424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,511,0.015520000209410986
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,511,0.017978666971127193
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,1023,0.022410665949185688
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,1023,0.020047999918460846
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,2047,0.03109866629044215
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,2047,0.034143999218940735
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,4095,0.06085333228111267
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,4095,0.05733866492907206
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,8191,0.09307733178138733
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,8191,0.10019733508427937
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,16383,0.17096000909805298
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,16383,0.18415466944376627
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,32767,0.3314773241678874
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,32767,0.35120534896850586
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,1,0.012250666817029318
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,1,0.013482666263977686
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,3,0.012154666086037954
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,65535,0.6508853435516357
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,3,0.013541333377361298
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,65535,0.6864746411641439
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,7,0.012047999848922094
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,255,0.6319839954376221
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,7,0.013445333888133367
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,fp8,255,0.6685280005137125
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,15,0.01249066616098086
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,15,0.013594667116800943
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,31,0.01231466606259346
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,31,0.01340266689658165
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,63,0.012309333930412928
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,127,0.012762666990359625
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,255,0.01569066693385442
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,255,0.014218666901191076
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,1023,0.028031999866167705
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,511,0.01605333387851715
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,511,0.018309333672126133
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,1023,0.02294933299223582
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,2047,0.04379733403523763
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,2047,0.03770133356253306
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,4095,0.07392533123493195
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,4095,0.05820799867312113
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,16383,0.2552906672159831
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,8191,0.13300266861915588
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,16383,0.1848479906717936
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,8191,0.10035733381907146
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,32767,0.49565335114796955
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,32767,0.3532693386077881
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,1,0.012106666962305704
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,65535,1.052191972732544
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,3,0.01163200040658315
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,65535,0.689903974533081
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,3,0.012874666601419449
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,7,0.012063999970753988
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,1,0.012655999511480331
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,15,0.011722666521867117
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,7,0.012762666990359625
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,15,0.012746666868527731
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,31,0.013295999417702356
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,31,0.01166933278242747
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,63,0.012080000092585882
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,63,0.012805332740147909
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,127,0.011887999872366587
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,127,0.014149333039919535
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,255,0.013365333278973898
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,255,0.01532799998919169
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,511,0.014815999815861383
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,511,0.017605333278576534
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,1023,0.01941866676012675
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,1023,0.022143999735514324
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,2047,0.03234666585922241
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,2047,0.028751999139785767
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,8191,0.08340266346931458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,4095,0.0516480008761088
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,4095,0.0481279989083608
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,8191,0.09928533434867859
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,16383,0.15422933300336203
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,16383,0.1843679944674174
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,32767,0.2952959934870402
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,32767,0.351034681002299
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,1,0.013274667163689932
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,1,0.011861333002646765
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,3,0.011813333878914515
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,3,0.013327999661366144
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,65535,0.5758080085118612
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,65535,0.6886826356252035
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,7,0.011744000017642975
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,7,0.013434667140245438
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,15,0.01184533288081487
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,15,0.012879999975363413
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,31,0.011855999628702799
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,63,0.01201066623131434
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,31,0.012874666601419449
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,63,0.013631999492645264
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,127,0.012128000458081564
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,127,0.014282666146755219
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,255,0.013754667093356451
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,255,0.015791999797026317
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,511,0.01562133307258288
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,511,0.0176959993938605
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,1023,0.02041600023706754
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,1023,0.022309333086013794
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,2047,0.03278400003910065
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,2047,0.03324799984693527
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,4095,0.05320000151793162
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,4095,0.05731200178464254
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,127,0.015146666516860327
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,8191,0.09322133660316467
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,63,0.01360000049074491
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,8191,0.10043199857076009
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,16383,0.15654399991035461
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,16383,0.18354666233062744
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,32767,0.2993066708246867
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,32767,0.3531573216120402
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,1,0.012538666526476542
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,1,0.013738666971524557
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,3,0.012389333297808966
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,65535,0.6540853182474772
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,3,0.013690666606028875
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,65535,0.6845173041025797
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,7,0.012202666451533636
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,7,0.013760000467300415
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,15,0.012517333030700684
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,15,0.01339200014869372
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,31,0.012351999680201212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,31,0.013530666629473368
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,63,0.013712000101804733
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,63,0.012554666648308435
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,127,0.012784000486135483
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,127,0.01526933287580808
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,255,0.014277332772811254
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,255,0.01578666642308235
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,511,0.016224000602960587
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,511,0.018511999398469925
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,1023,0.0281333327293396
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,2047,0.04349866509437561
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,1023,0.0234400009115537
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,2047,0.03808533400297165
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,4095,0.07400000095367432
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,4095,0.05884799857934316
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,8191,0.13505599896113077
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,8191,0.1009333332379659
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,16383,0.24921600023905435
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,16383,0.184879998366038
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,1,0.011301333705584208
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,32767,0.35248533884684247
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,1,0.012495999534924826
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,32767,0.49533867835998535
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,3,0.010863999525705973
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,7,0.011205332974592844
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,7,0.01209066684047381
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,65535,1.0533599853515625
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,15,0.011168000598748526
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,3,0.012367999802033106
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,65535,0.6940266291300455
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,31,0.011007999380429586
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,15,0.012341332932313284
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,31,0.012346666306257248
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,63,0.01101333275437355
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,127,0.011039999624093374
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,63,0.01240533341964086
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,127,0.0145066666106383
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,255,0.014362666755914688
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,255,0.013178666432698568
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,511,0.013349333157142004
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,511,0.0143306665122509
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,1023,0.013354666531085968
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,1023,0.014522666732470194
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,2047,0.014501333236694336
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,2047,0.015658666690190632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,4095,0.015824000040690105
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,4095,0.016666666915019352
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,8191,0.017957333475351334
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,8191,0.019354666272799175
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,16383,0.02236266682545344
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,16383,0.024773334463437397
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,32767,0.03205333401759466
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,32767,0.03488533447186152
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,65535,0.05054399867852529
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,65535,0.05606933434804281
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,131071,0.08965866764386494
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,131071,0.09847467144330342
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,7,0.012202666451533636
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,1,0.011098666737476984
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,1,0.011994666109482447
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,3,0.011125333607196808
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,3,0.012223999947309494
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,63,0.011077333241701126
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,7,0.011050666371981302
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,15,0.01098666712641716
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,15,0.012362666428089142
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,31,0.011098666737476984
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,31,0.012442667037248611
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,63,0.012085333466529846
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,127,0.011109333485364914
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,127,0.014432000617186228
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,255,0.013487999637921652
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,255,0.014229333649079004
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,511,0.013546666751305262
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,511,0.014389333625634512
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,1023,0.01357866699496905
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,1023,0.015226667126019796
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,2047,0.014815999815861383
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,2047,0.016384000579516094
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,4095,0.016544000556071598
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,4095,0.01732800031701724
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,8191,0.01858666663368543
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,8191,0.02027733375628789
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,16383,0.023391999304294586
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,16383,0.02571733295917511
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,32767,0.0330826664964358
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,32767,0.035760000348091125
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,65535,0.053786665201187134
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,65535,0.05839466551939646
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,131071,0.09083732962608337
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,131071,0.10654933253924052
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,1,0.011136000355084738
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,1,0.012096000214417776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,3,0.011391999820868174
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,3,0.011989332735538483
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,7,0.011274666835864386
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,7,0.012202666451533636
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,15,0.011370666325092316
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,15,0.012128000458081564
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,31,0.011050666371981302
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,31,0.012383999923865
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,63,0.011087999989589056
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,63,0.013023999830087027
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,127,0.011231999844312668
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,127,0.014479999740918478
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,255,0.01341333364446958
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,255,0.014522666732470194
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,511,0.013936000565687815
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,511,0.01552533358335495
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,1023,0.014170666535695394
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,1023,0.015637333194414776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,2047,0.016415999581416447
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,2047,0.017050666113694508
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,4095,0.016949333250522614
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,4095,0.01793066660563151
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,8191,0.018944000204404194
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,8191,0.020373333245515823
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,16383,0.02372266600529353
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,16383,0.0258240004380544
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,131071,0.09386666615804036
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,32767,0.036501333117485046
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,32767,0.03676266719897588
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,65535,0.057722667853037514
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,65535,0.06256533165772755
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,131071,0.10788266857465108
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,1,0.010944000134865442
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,1,0.012213333199421564
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,3,0.010944000134865442
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,3,0.012175999581813812
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,7,0.01108266661564509
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,7,0.01202133297920227
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,15,0.010911999891201654
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,15,0.012351999680201212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,31,0.011071999867757162
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,31,0.012069333344697952
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,63,0.010933333386977514
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,63,0.01221866657336553
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,127,0.01102399950226148
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,127,0.014256000518798828
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,255,0.013424000392357508
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,255,0.014511999984582266
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,511,0.013381333400805792
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,511,0.01470400020480156
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,1023,0.01461333284775416
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,1023,0.015642666568358738
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,2047,0.015728000551462173
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,2047,0.016538667182127636
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,4095,0.01821333294113477
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,4095,0.01930133377512296
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,8191,0.021333334346612293
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,8191,0.023887999355793
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,16383,0.029909332593282063
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,65535,0.09303999940554301
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,16383,0.034154665966828666
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,32767,0.04790933430194855
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,32767,0.05301333467165629
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,65535,0.08431466420491536
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,131071,0.1536853313446045
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,131071,0.18454400698343912
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,1,0.011328000575304031
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,1,0.012560000022252401
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,3,0.011071999867757162
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,3,0.012138667205969492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,7,0.011237333218256632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,7,0.0124746672809124
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,15,0.011114666859308878
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,15,0.01249066616098086
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,31,0.011231999844312668
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,31,0.012448000411192576
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,63,0.011146667102972666
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,63,0.012213333199421564
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,127,0.011130666981140772
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,127,0.01488000030318896
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,255,0.013487999637921652
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,255,0.014554666976133982
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,511,0.01369599997997284
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,511,0.015386667102575302
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,1023,0.01469333345691363
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,1023,0.016634666671355564
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,2047,0.016842667013406754
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,2047,0.017994667092959087
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,4095,0.01883200059334437
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,4095,0.020197333147128422
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,8191,0.022154666483402252
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,8191,0.024559999505678814
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,16383,0.030960001051425934
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,16383,0.034847999612490334
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,32767,0.05144000053405762
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,32767,0.053914666175842285
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,65535,0.08551466464996338
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,65535,0.10154666503270467
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,131071,0.15573333700497946
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,131071,0.1856000026067098
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,1,0.011429333438475927
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,1,0.012778667112191519
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,3,0.011343999455372492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,3,0.01250133290886879
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,7,0.011541333049535751
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,7,0.012346666306257248
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,63,0.01139733319481214
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,15,0.01139733319481214
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,15,0.012373333175977072
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,31,0.011551999797423681
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,31,0.012261333564917246
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,63,0.01250133290886879
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,127,0.011461333682139715
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,127,0.014709333578745524
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,255,0.013701333353916803
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,255,0.015306666493415833
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,511,0.01462399959564209
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,511,0.015509333461523056
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,1023,0.016528000434239704
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,1023,0.01669866715868314
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,2047,0.017008000363906223
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,2047,0.017802666872739792
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,4095,0.018944000204404194
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,4095,0.02089600016673406
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,8191,0.02248000105222066
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,8191,0.025029333929220837
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,16383,0.03365866591533025
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,16383,0.03629866739114126
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,32767,0.055071999629338585
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,32767,0.05948266883691152
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,65535,0.1014453371365865
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,1,0.012901333471139273
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,65535,0.09458133578300476
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,131071,0.1729173262914022
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,131071,0.18603734175364176
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,1,0.013466666142145792
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,3,0.012874666601419449
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,3,0.013487999637921652
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,7,0.01310933381319046
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,7,0.01360000049074491
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,31,0.013530666629473368
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,15,0.012613333761692047
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,15,0.013471999516089758
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,31,0.012810666114091873
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,63,0.012879999975363413
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,63,0.013487999637921652
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,127,0.012928000340859095
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,127,0.015200000256299973
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,255,0.014309333016475042
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,255,0.015365333606799444
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,511,0.016106666376193363
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,511,0.018538666268189747
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,1023,0.021226666867733
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,1023,0.022805333137512207
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,2047,0.032111999889214836
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,2047,0.03323200096686681
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,8191,0.09191999832789104
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,4095,0.0544053316116333
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,8191,0.1011786659558614
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,4095,0.05769599974155426
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,16383,0.17821866273880005
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,16383,0.18516800800959268
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,1,0.013424000392357508
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,32767,0.299834668636322
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,3,0.01314666618903478
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,7,0.01312000056107839
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,7,0.013829333086808523
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,15,0.012938667088747025
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,15,0.013957332819700241
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,31,0.01302933320403099
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,3,0.013850666582584381
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,1,0.01369599997997284
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,63,0.013056000073750814
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,32767,0.35531731446584064
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,31,0.014314666390419006
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,63,0.013898666948080063
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,127,0.01322666679819425
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,127,0.015354666858911514
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,255,0.015381333728631338
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,255,0.016149333367745083
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,511,0.017018667111794155
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,511,0.018730666488409042
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,1023,0.028922667105992634
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,1023,0.0236160010099411
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,4095,0.07517333328723907
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,4095,0.05931733548641205
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,8191,0.13596266508102417
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,2047,0.03829866647720337
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,2047,0.04451733330885569
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,8191,0.10268800457318623
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,16383,0.1863200068473816
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,3,0.0169813334941864
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,16383,0.25514666239420575
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,3,0.01801066721479098
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,32767,0.4984906514485677
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,1,0.0183146670460701
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,1,0.017504000415404636
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,7,0.01711999997496605
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,7,0.01838933303952217
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,15,0.01717866708834966
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,32767,0.3570133447647095
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,31,0.017263999829689663
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,31,0.01811733345190684
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,63,0.017360000560681026
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,15,0.01828266680240631
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,63,0.018357332795858383
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,127,0.017525333911180496
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,255,0.0207893339296182
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,255,0.022405333817005157
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,127,0.021397332350413006
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,511,0.03128000100453695
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,511,0.027823999524116516
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,2047,0.07645866771539052
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,1023,0.04574400186538696
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,1023,0.042394667863845825
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,2047,0.0636106679836909
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,4095,0.13426132996877035
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,4095,0.10539199908574422
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,8191,0.19058134158452353
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,1,0.010805333654085795
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,8191,0.251525342464447
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,1,0.012037333101034164
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,3,0.010725333044926325
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,3,0.011600000162919363
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,7,0.010847999403874079
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,7,0.011546666423479715
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,15,0.011066666493813196
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,15,0.012015999605258306
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,31,0.011226666470368704
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,31,0.012005332857370377
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,16383,0.35605867703755695
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,127,0.011109333485364914
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,16383,0.4905386765797933
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,63,0.011744000017642975
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,32767,0.6934933662414551
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,63,0.010821333775917688
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,127,0.014127999544143677
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,32767,1.0437706311543782
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,255,0.014650666465361914
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,255,0.013066666821638743
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,511,0.015279999623696009
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,511,0.014357333381970724
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,1023,0.016399999459584553
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,8191,0.029850666721661884
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,1023,0.015482666591803232
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,2047,0.01685333376129468
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,4095,0.02123733361562093
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,2047,0.01897066707412402
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,4095,0.023658665517965954
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,8191,0.03350399931271871
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,16383,0.04794133206208547
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,16383,0.05287999908129374
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,32767,0.08380267024040222
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,32767,0.09485333164532979
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,65535,0.1536853313446045
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,65535,0.1853440006573995
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,131071,0.2914133270581563
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,131071,0.35396798451741535
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,1,0.011034666250149408
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,1,0.01190399999419848
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,3,0.010922666639089584
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,3,0.01180800050497055
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,7,0.011029332876205444
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,7,0.011957333733638128
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,15,0.011098666737476984
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,15,0.011781333635250727
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,127,0.01121066634853681
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,31,0.01098666712641716
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,31,0.011823999385039011
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,63,0.011007999380429586
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,63,0.011813333878914515
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,127,0.014138666292031607
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,255,0.01333333303531011
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,255,0.014778666198253632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,511,0.014698666830857595
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,511,0.0164533331990242
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,1023,0.016255999604860943
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,1023,0.016965333372354507
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,2047,0.017162666966517765
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,2047,0.019648000597953796
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,4095,0.021520001192887623
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,4095,0.024298667907714844
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,8191,0.030597334106763203
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,8191,0.03417066733042399
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,16383,0.05017066498597463
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,16383,0.053488001227378845
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,32767,0.08593066533406575
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,32767,0.10094933708508809
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,65535,0.1553600033124288
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,65535,0.18482667207717896
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,1,0.011359999577204386
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,131071,0.2954933245976766
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,131071,0.3531839847564697
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,1,0.012138667205969492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,3,0.011237333218256632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,3,0.012165332833925882
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,7,0.011258666714032492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,7,0.012223999947309494
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,15,0.011194666226704916
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,15,0.01231466606259346
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,63,0.012362666428089142
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,31,0.011173332730929056
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,31,0.012453333785136541
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,63,0.01126933346192042
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,127,0.01110400011142095
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,127,0.015072000523408255
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,255,0.014271999398867289
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,255,0.015541333705186844
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,511,0.015573333948850632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,511,0.016645333419243496
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,1023,0.01672533278663953
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,1023,0.01700266698996226
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,2047,0.017312000195185345
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,2047,0.019845332950353622
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,4095,0.022240000466505688
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,4095,0.024634666740894318
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,8191,0.038848000268141426
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,8191,0.03555200000603994
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,16383,0.05907199780146281
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,16383,0.05413866539796194
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,32767,0.086325337489446
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,32767,0.10162666440010071
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,65535,0.15898133317629495
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,65535,0.184714674949646
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,131071,0.333077331384023
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,131071,0.351088007291158
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,1,0.014170666535695394
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,1,0.01524266724785169
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,3,0.014287999520699183
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,3,0.015050667027632395
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,7,0.015669333438078564
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,7,0.014362666755914688
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,15,0.014159999787807465
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,15,0.01532799998919169
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,31,0.014303999642531076
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,31,0.014901333798964819
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,63,0.014405333747466406
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,63,0.01534933348496755
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,127,0.0145066666106383
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,127,0.016794666647911072
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,255,0.01626666635274887
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,255,0.01777600000301997
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,511,0.01884799947341283
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,511,0.01987733319401741
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,1023,0.030192000170548756
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,1023,0.024847999215126038
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,2047,0.04539733131726583
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,2047,0.03996799886226654
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,4095,0.07784000039100647
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,4095,0.06122133135795593
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,8191,0.10523200035095215
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,8191,0.13768000404040018
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,16383,0.25833600759506226
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,1,0.019093333433071773
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,7,0.017674667139848072
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,16383,0.18976000944773355
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,7,0.01893866683046023
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,1,0.017605333278576534
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,15,0.01794133335351944
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,15,0.019071999937295914
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,3,0.01766933376590411
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,31,0.017621333400408428
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,3,0.018944000204404194
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,31,0.01883200059334437
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,63,0.017850667238235474
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,63,0.019088000059127808
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,127,0.018112000077962875
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,127,0.02222399910291036
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,255,0.021914665897687275
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,255,0.02334933231274287
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,511,0.03287466615438461
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,511,0.028570666909217834
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,1023,0.048895999789237976
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,1023,0.04354666670163473
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,4095,0.13796266913414001
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,2047,0.07821333408355713
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,4095,0.10654933253924052
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,2047,0.06552533308664958
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,8191,0.2550826668739319
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,1,0.024469333390394848
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,8191,0.19269333283106485
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,3,0.024256000916163128
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,1,0.02610666553179423
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,7,0.025989333788553875
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,7,0.024832000335057575
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,16383,0.36352535088857013
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,15,0.02495466669400533
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,3,0.026591998835404713
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,15,0.02619733413060506
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,16383,0.4939519961675008
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,31,0.02465066562096278
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,31,0.025983999172846477
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,63,0.026464000344276428
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,127,0.025706666211287182
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,63,0.02499733368555705
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,127,0.030986666679382324
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,255,0.03453333427508672
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,511,0.051632001996040344
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,511,0.05042133231957754
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,255,0.03775466730197271
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,2047,0.13788800438245138
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,2047,0.1136853297551473
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,1023,0.07075733443101247
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,1023,0.08095466593901317
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,4095,0.25325334072113037
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,8191,0.36457598209381104
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,4095,0.19632534186045328
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,3,0.02015999952952067
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,8191,0.48841599623362225
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,fp8,3,0.021141332884629566
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,16383,1.0398826599121094
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,7,0.019823999454577763
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,fp8,1,0.021338666478792827
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,fp8,7,0.021141332884629566
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,1,0.02004266654451688
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,fp8,15,0.02109866589307785
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,15,0.019493332753578823
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,16383,0.7043413321177164
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,fp8,31,0.02128000060717265
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,63,0.01993600030740102
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,fp8,63,0.021503999829292297
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,127,0.02012266715367635
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,fp8,127,0.023797333240509033
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,255,0.023157333334287006
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,31,0.019946667055288952
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,511,0.03514133393764496
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,fp8,511,0.031632001201311745
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,fp8,1023,0.046223998069763184
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,1023,0.05110399921735128
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,2047,0.08118933439254761
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,fp8,2047,0.06791999936103821
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,1,0.02629333237806956
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,fp8,1,0.02805333336194356
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,fp8,255,0.025600001215934753
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,fp8,7,0.02815466622511546
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,15,0.02619733413060506
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,3,0.026213333010673523
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,fp8,15,0.027893332143624622
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,7,0.02590399980545044
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,fp8,3,0.0278613343834877
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,fp8,4095,0.1102239986260732
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,4095,0.1402293344338735
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,31,0.026341333985328674
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,fp8,31,0.028031999866167705
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,63,0.02645866572856903
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,fp8,63,0.027999999622503918
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,127,0.026421333352724712
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,fp8,127,0.03266133368015289
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,511,0.05526933570702871
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,fp8,511,0.05286933481693268
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,1023,0.08345599969228108
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,fp8,1023,0.07338666419188182
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,4095,0.25862399737040204
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,fp8,4095,0.2007946570714315
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,fp8,255,0.03623999903599421
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,2047,0.14079466462135315
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,255,0.04062933226426443
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,3,0.039120001097520195
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,fp8,3,0.04213866591453552
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,fp8,2047,0.11628266175587972
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,7,0.03908800085385641
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,1,0.039093332986036934
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,fp8,15,0.0429066667954127
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,15,0.039477333426475525
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,31,0.03930133332808813
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,63,0.03935466706752777
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,fp8,63,0.04214933514595032
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,fp8,127,0.051125332713127136
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,127,0.045696000258127846
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,255,0.058970664938290916
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,fp8,1,0.04199466605981191
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,fp8,255,0.0637546678384145
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,2047,0.2601173321406047
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,fp8,7,0.0420959989229838
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,4095,0.4927733341852824
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,fp8,511,0.08673600355784099
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,fp8,31,0.04177066683769226
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,511,0.08943466345469157
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,fp8,2047,0.21235199769337973
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,fp8,4095,0.37930134932200116
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,1,0.03125333289305369
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,fp8,1023,0.12682666381200156
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,1023,0.14494933684666952
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,fp8,1,0.03311466674009959
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,3,0.03166933357715607
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,7,0.03150933235883713
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,fp8,7,0.03312533348798752
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,fp8,3,0.033071999748547874
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,15,0.03164266546567281
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,31,0.031471999982992806
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,fp8,31,0.03301866600910822
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,63,0.031845333675543465
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,fp8,63,0.0332640012105306
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,127,0.03212266663710276
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,fp8,15,0.03332799921433131
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,fp8,127,0.03719999889532725
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,255,0.04649066428343455
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,fp8,255,0.04197333256403605
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,511,0.06113066772619883
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,fp8,511,0.05938133100668589
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,1023,0.09032533566157024
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,fp8,1023,0.08017600079377492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,fp8,1,0.04587733248869578
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,2047,0.14923733472824097
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,3,0.04297600189844767
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,fp8,2047,0.123471995194753
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,fp8,3,0.04590400060017904
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,7,0.04341333111127218
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,fp8,7,0.0458133320013682
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,15,0.04340266684691111
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,31,0.04312533140182495
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,63,0.04368533194065094
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,fp8,63,0.04614933331807455
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,127,0.05158400038878123
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,fp8,127,0.054757331808408104
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,fp8,15,0.04614933331807455
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,255,0.06610666712125142
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,fp8,31,0.04587199787298838
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,fp8,255,0.06931733091672261
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,511,0.09609599908192952
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,1,0.043141335248947144
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,fp8,511,0.09311466415723164
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,1,0.06962133447329204
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,1023,0.1518346667289734
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,3,0.06929600238800049
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,fp8,1,0.07587733368078868
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,7,0.06946133573849995
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,fp8,1023,0.13363200426101685
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,fp8,3,0.0748586654663086
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,15,0.06985599795977275
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,fp8,15,0.07569600145022075
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,31,0.07060799996058147
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,fp8,7,0.07492266595363617
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,2047,0.267247994740804
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,63,0.07604266703128815
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,fp8,63,0.07506133119265239
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,fp8,2047,0.21985065937042236
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,127,0.0794239987929662
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,fp8,31,0.0751200020313263
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,fp8,127,0.09675733248392741
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,1023,0.2709173361460368
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,255,0.1043839951356252
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,fp8,511,0.1600160002708435
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,511,0.164000004529953
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,2047,0.5016533136367798
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,1,0.011941333611806234
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,7,0.012074666718641916
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,7,0.011055999745925268
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,15,0.010965333630641302
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,15,0.012272000312805176
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,fp8,2047,0.4111093282699585
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,31,0.011002667248249054
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,31,0.012245333443085352
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,63,0.012175999581813812
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,127,0.011087999989589056
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,127,0.013157332936922709
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,255,0.012378666549921036
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,255,0.015157333264748255
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,63,0.01110400011142095
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,511,0.014906667172908783
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,511,0.01634666696190834
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,1023,0.016901332885026932
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,1023,0.01950399950146675
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,2047,0.02107200026512146
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,2047,0.023573334018389385
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,fp8,1023,0.24040534098943075
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,4095,0.029904000461101532
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,4095,0.0335413341720899
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,3,0.012069333344697952
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,3,0.010981333752473196
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,8191,0.04789333542188009
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,fp8,255,0.1129813293615977
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,16383,0.08468799789746602
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,32767,0.15413332978884378
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,32767,0.1838080088297526
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,16383,0.09682133793830872
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,65535,0.29173866907755536
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,1,0.011125333607196808
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,1,0.011098666737476984
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,131071,0.572218656539917
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,1,0.012282667060693106
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,131071,0.6911359628041586
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,3,0.011450666934251785
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,7,0.012234666695197424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,3,0.011823999385039011
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,7,0.011152000476916632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,65535,0.354362686475118
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,15,0.011359999577204386
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,15,0.01231466606259346
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,31,0.011317333827416102
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,8191,0.05330666899681091
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,31,0.011952000359694162
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,63,0.011258666714032492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,63,0.01211200033624967
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,127,0.011402666568756104
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,127,0.013343999783198038
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,255,0.01293333371480306
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,255,0.016437333077192307
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,511,0.015546667079130808
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,511,0.01714666684468587
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,1023,0.016986666868130367
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,1023,0.019472000499566395
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,2047,0.02170666555563609
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,2047,0.02446399877468745
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,4095,0.03163733333349228
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,4095,0.03376533339420954
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,8191,0.0497920016447703
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,32767,0.15600533286730447
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,8191,0.05514133473237356
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,16383,0.08548800150553386
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,16383,0.10342933734258015
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,32767,0.18573333819707236
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,65535,0.2954240043958028
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,65535,0.35201601187388104
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,131071,0.6897386709849039
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,1,0.011701333026091257
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,131071,0.5782026847203573
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,1,0.012794667234023413
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,3,0.011893333246310553
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,3,0.012794667234023413
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,7,0.01166933278242747
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,15,0.011637333780527115
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,7,0.01257066677014033
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,15,0.012917333592971167
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,31,0.01191466674208641
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,31,0.012768000364303589
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,63,0.011973333855470022
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,63,0.012869333227475485
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,127,0.011829332758982977
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,127,0.014469332993030548
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,255,0.013130666067202887
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,255,0.016645333419243496
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,511,0.016058667252461117
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,511,0.01756799966096878
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,1023,0.017957333475351334
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,1023,0.019893333315849304
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,2047,0.022181332111358643
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,2047,0.024901332954565685
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,4095,0.03419200082619985
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,4095,0.03526400029659271
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,8191,0.05515199899673462
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,8191,0.058864002426465355
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,16383,0.1048959990342458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,16383,0.10202667117118835
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,32767,0.1734293301900228
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,32767,0.18609066804250082
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,65535,0.4001760085423787
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,65535,0.35228268305460614
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,131071,0.7849706808725992
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,131071,0.683685302734375
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,fp8,1,0.03877866764863332
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,1,0.03844800094763438
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,3,0.038047999143600464
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,fp8,3,0.03905066599448522
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,7,0.037861332297325134
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,fp8,7,0.03898133337497711
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,15,0.038191998998324074
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,fp8,15,0.03898133337497711
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,31,0.03830400109291077
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,fp8,31,0.03884266565243403
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,fp8,63,0.03904533386230469
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,63,0.03818666686614355
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,fp8,127,0.04833066463470459
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,127,0.04413333535194397
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,255,0.05633600056171417
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,fp8,255,0.06239999830722809
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,511,0.08478933572769165
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,1,0.06440000236034393
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,fp8,1,0.06326933205127716
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,3,0.06443733473618825
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,fp8,3,0.06248533229033152
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,fp8,511,0.08623466889063518
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,1023,0.14241600036621094
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,fp8,1023,0.1250986655553182
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,7,0.06431999802589417
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,fp8,7,0.06302399933338165
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,15,0.06444266438484192
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,fp8,15,0.063360000650088
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,31,0.0654666672150294
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,fp8,31,0.06333333253860474
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,127,0.07331733405590057
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,fp8,127,0.08754133184750874
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,255,0.09521067142486572
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,fp8,255,0.10483200351397197
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,fp8,63,0.06354666749636333
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,63,0.07042666773001353
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,511,0.1487626632054647
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,fp8,511,0.15221866965293884
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,fp8,1,0.11326932907104492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,1,0.11529067158699036
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,3,0.11569600303967793
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,fp8,3,0.1132480005423228
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,fp8,1023,0.23189334074656168
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,1023,0.26585066318511963
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,fp8,7,0.11377066373825073
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,15,0.11752000451087952
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,fp8,15,0.11365866661071777
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,31,0.125408003727595
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,fp8,31,0.11492266257603963
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,127,0.12997333208719888
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,fp8,127,0.15683199961980185
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,255,0.17361599206924438
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,fp8,255,0.18588266770044962
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,7,0.11656000216801961
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,fp8,63,0.12358400225639343
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,63,0.12618133425712585
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,fp8,1,0.06575466692447662
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,511,0.28281599283218384
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,1,0.06490666667620341
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,fp8,511,0.2794453303019206
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,3,0.06498666604359944
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,fp8,3,0.06560533245404561
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,fp8,1023,0.43703468640645343
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,1023,0.5519573291142782
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,7,0.06572799881299336
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,15,0.06506666541099548
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,31,0.06705066561698914
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,fp8,31,0.06633066634337108
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,fp8,7,0.06549866497516632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,fp8,15,0.06609599788983662
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,fp8,127,0.08867200215657552
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,127,0.07489599784215291
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,255,0.09679466485977173
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,fp8,255,0.11074666182200114
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,1,0.1165706713994344
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,fp8,1,0.11433600385983785
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,3,0.11635733644167583
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,fp8,3,0.11361599961916606
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,63,0.07193600138028462
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,fp8,63,0.0692853331565857
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,511,0.1509226659933726
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,fp8,511,0.15705600380897522
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,7,0.11646933356920879
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,fp8,7,0.11443199714024861
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,15,0.11926399668057759
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,fp8,15,0.11404800415039062
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,fp8,31,0.11498666803042094
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,63,0.1269599994023641
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,fp8,63,0.12477333347002666
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,127,0.13185066978136697
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,fp8,127,0.15941866238911948
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,255,0.17407466967900595
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,fp8,255,0.1919999917348226
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,31,0.12634666760762533
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,1,0.21889066696166992
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,511,0.27849600712458294
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,fp8,1,0.21453867355982462
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,fp8,511,0.2834453384081523
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,3,0.2208319902420044
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,fp8,3,0.2146773338317871
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,7,0.22362667322158813
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,fp8,7,0.2145599921544393
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,15,0.23549866676330566
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,31,0.2367039918899536
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,63,0.24023999770482382
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,fp8,15,0.21715199947357178
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,fp8,63,0.2305813431739807
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,127,0.24620266755421957
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,fp8,127,0.2996586759885152
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,1,0.011109333485364914
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,3,0.012229333321253458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,7,0.011242666592200598
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,7,0.012250666817029318
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,fp8,31,0.22819199164708456
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,15,0.011322667201360067
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,15,0.012485332787036896
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,31,0.012213333199421564
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,63,0.011242666592200598
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,1,0.012250666817029318
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,63,0.012293333808581034
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,31,0.01145600030819575
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,127,0.011264000087976456
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,127,0.013429333766301474
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,255,0.012671999633312225
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,255,0.0143306665122509
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,511,0.014560000350077948
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,511,0.017071999609470367
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,3,0.011488000551859537
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,1023,0.018874666343132656
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,1023,0.02160000056028366
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,2047,0.02737066646416982
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,2047,0.0320266659061114
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,4095,0.045647998650868736
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,8191,0.08167999982833862
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,4095,0.05106133222579956
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,16383,0.15058133006095886
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,8191,0.09291733304659526
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,16383,0.18262400229771933
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,32767,0.28904000918070477
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,32767,0.3511626720428467
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,65535,0.5704799890518188
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,65535,0.6835040251413981
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,1,0.011877333124478659
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,131071,1.130346695582072
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,1,0.012746666868527731
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,131071,1.354581356048584
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,3,0.011898666620254517
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,3,0.012815999488035837
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,255,0.3290773431460063
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,7,0.011765333513418833
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,7,0.012906666845083237
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,fp8,255,0.3562186559041341
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,15,0.011690666278203329
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,15,0.012768000364303589
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,31,0.01156266654531161
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,63,0.011968000481526056
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,127,0.011999999483426413
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,127,0.014682666709025701
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,255,0.013658666362365087
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,255,0.015024000157912573
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,511,0.015173333386580149
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,511,0.017797333498795826
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,1023,0.01941866676012675
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,2047,0.028255999088287354
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,2047,0.03214933226505915
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,1023,0.0227360005180041
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,4095,0.048026666045188904
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,8191,0.08303466439247131
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,4095,0.052970667680104576
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,8191,0.09920000036557515
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,16383,0.15355199575424194
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,16383,0.1821440060933431
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,32767,0.2943840026855469
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,32767,0.3516853253046672
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,fp8,511,0.5402773221333822
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,511,0.5825920104980469
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,65535,0.5752053260803223
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,65535,0.6809013684590658
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,131071,1.1388160387674968
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,1,0.012506666282812754
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,3,0.013242666920026144
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,7,0.012986666212479273
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,7,0.011930666863918304
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,15,0.011994666109482447
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,131071,1.3550559679667156
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,15,0.012917333592971167
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,31,0.012170666207869848
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,31,0.012901333471139273
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,63,0.013023999830087027
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,31,0.013253333667914072
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,63,0.011968000481526056
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,127,0.012138667205969492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,127,0.014922666052977243
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,255,0.013914667069911957
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,255,0.015439999600251516
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,511,0.015477333217859268
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,511,0.018570666511853535
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,2047,0.032933334509531655
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,1023,0.022874665757020313
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,1023,0.019909333437681198
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,2047,0.03311466674009959
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,4095,0.052655999859174095
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,4095,0.0572213331858317
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,8191,0.09186666210492452
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,8191,0.09892266988754272
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,16383,0.2067626714706421
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,16383,0.18343466520309448
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,32767,0.35143999258677167
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,32767,0.2983679970105489
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,65535,0.6852320035298666
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,65535,0.5756533145904541
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,1,0.012383999923865
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,1,0.012800000607967377
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,131071,1.3530666033426921
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,3,0.013013333082199097
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,3,0.012085333466529846
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,7,0.011706666400035223
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,131071,1.2923999627431233
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,15,0.01166933278242747
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,7,0.012693333129088083
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,15,0.012719999998807907
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,31,0.011701333026091257
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,63,0.013189333180586496
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,63,0.012698666503032049
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,63,0.012085333466529846
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,127,0.01227733368674914
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,127,0.014394666999578476
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,31,0.012906666845083237
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,255,0.013450667262077332
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,255,0.014597332725922266
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,511,0.015295999745527903
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,511,0.017594666530688603
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,1023,0.019706666469573975
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,2047,0.02861333390076955
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,1023,0.02250666668017705
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,2047,0.03203733265399933
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,4095,0.048010667165120445
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,4095,0.05285866558551788
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,8191,0.08388800422350566
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,8191,0.10071999828020732
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,16383,0.15313067038853964
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,16383,0.1844586730003357
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,32767,0.3540000120798747
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,32767,0.2953386704126994
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,1,0.012495999534924826
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,65535,0.6894506613413492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,65535,0.574351986249288
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,1,0.013530666629473368
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,131071,1.138474702835083
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,7,0.012085333466529846
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,3,0.01184533288081487
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,131071,1.3627840677897136
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,15,0.012229333321253458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,3,0.013381333400805792
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,7,0.013536000003417334
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,15,0.012800000607967377
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,31,0.012037333101034164
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,31,0.01351999988158544
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,1,0.01314666618903478
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,63,0.011802667131026586
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,63,0.012970666090647379
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,3,0.012170666207869848
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,127,0.011909333368142446
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,127,0.014655999839305878
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,255,0.014159999787807465
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,255,0.014991999914248785
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,511,0.015125333021084467
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,511,0.017642666896184284
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,1023,0.020506666352351505
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,1023,0.022687998910744984
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,8191,0.0935040016969045
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,2047,0.03253866732120514
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,2047,0.03281066566705704
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,4095,0.053413331508636475
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,4095,0.05677866439024607
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,8191,0.09987200299898784
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,32767,0.2990880012512207
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,16383,0.19987734158833823
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,16383,0.18403732776641846
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,32767,0.350597341855367
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,1,0.011418666690587997
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,1,0.012149333953857422
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,3,0.011205332974592844
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,65535,0.5842399994532267
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,65535,0.6874399979909261
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,7,0.01091733326514562
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,3,0.01209066684047381
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,7,0.012085333466529846
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,15,0.011114666859308878
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,131071,1.2967039744059246
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,15,0.01198400060335795
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,131071,1.3565386136372883
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,31,0.011221333096424738
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,31,0.012037333101034164
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,63,0.010965333630641302
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,63,0.01219733307758967
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,127,0.011231999844312668
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,127,0.014666666587193808
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,255,0.01357866699496905
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,511,0.013760000467300415
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,255,0.014495999862750372
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,511,0.014368000129858652
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,1023,0.013834666460752487
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,1023,0.014698666830857595
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,2047,0.014773332824309668
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,2047,0.015706667055686314
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,4095,0.01590399940808614
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,4095,0.016805333395799
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,8191,0.01821333294113477
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,8191,0.019487999379634857
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,16383,0.02279466638962428
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,65535,0.05606933434804281
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,16383,0.024911999702453613
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,32767,0.0324799989660581
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,32767,0.03492266684770584
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,65535,0.050586665670077004
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,131071,0.08947199583053589
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,3,0.011909333368142446
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,131071,0.0983786682287852
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,1,0.010928000013033548
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,1,0.012383999923865
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,3,0.010965333630641302
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,7,0.010960000256697336
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,7,0.01191466674208641
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,15,0.011152000476916632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,15,0.012037333101034164
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,31,0.011071999867757162
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,31,0.012453333785136541
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,63,0.011306667079528173
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,63,0.012223999947309494
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,127,0.011178666104873022
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,127,0.01452800010641416
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,255,0.013642666240533194
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,255,0.014394666999578476
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,511,0.013807999591032663
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,511,0.014650666465361914
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,1023,0.013957332819700241
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,1023,0.015285332997639975
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,2047,0.015013333410024643
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,2047,0.01653333380818367
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,4095,0.016607999801635742
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,4095,0.01764800027012825
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,8191,0.019002666076024372
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,8191,0.020362666497627895
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,16383,0.023498666783173878
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,16383,0.025813333690166473
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,32767,0.03321066747109095
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,32767,0.03568533311287562
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,65535,0.05380799869696299
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,65535,0.056976000467936196
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,131071,0.09141332904497783
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,131071,0.10780800382296245
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,1,0.010869332899649939
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,1,0.012037333101034164
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,3,0.010960000256697336
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,3,0.01202133297920227
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,7,0.011061333119869232
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,7,0.012335999558369318
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,15,0.01098666712641716
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,15,0.012165332833925882
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,31,0.011029332876205444
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,31,0.0122079998254776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,63,0.01090666651725769
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,63,0.0122079998254776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,127,0.01119999960064888
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,127,0.015029333531856537
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,255,0.013637332866589228
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,255,0.014495999862750372
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,511,0.013642666240533194
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,511,0.014560000350077948
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,1023,0.014917333920796713
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,1023,0.015935999651749928
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,2047,0.01607999950647354
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,2047,0.01681600014368693
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,4095,0.01829333355029424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,4095,0.019621333728233974
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,8191,0.021295999487241108
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,8191,0.023898666103680927
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,16383,0.03443733354409536
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,16383,0.030245333909988403
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,32767,0.04831466575463613
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,32767,0.053317333261171974
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,65535,0.08458667000134786
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,65535,0.09346133470535278
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,3,0.011333333949247995
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,131071,0.15424000223477682
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,131071,0.18227734168370566
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,1,0.01138666644692421
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,1,0.012015999605258306
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,3,0.012117333710193634
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,7,0.011114666859308878
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,7,0.012319999436537424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,15,0.011514666179815928
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,15,0.012272000312805176
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,31,0.011365332951148352
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,31,0.012421333541472753
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,63,0.011178666104873022
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,63,0.012261333564917246
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,127,0.01109333336353302
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,127,0.014842666685581207
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,255,0.013989333063364029
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,255,0.01463466634353002
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,511,0.013914667069911957
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,511,0.015573333948850632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,1023,0.015135999768972397
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,1023,0.017018667111794155
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,2047,0.016858667135238647
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,2047,0.017658667018016178
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,4095,0.018960000326236088
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,4095,0.020351999749739964
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,8191,0.022197333474953968
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,8191,0.025397333006064098
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,16383,0.031141333281993866
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,16383,0.03482133398453394
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,32767,0.050373335679372154
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,32767,0.05485333502292633
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,65535,0.08656000097592671
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,65535,0.10139200091362
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,131071,0.1560640037059784
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,131071,0.18450667460759482
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,1,0.012800000607967377
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,1,0.01309866706530253
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,3,0.012906666845083237
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,3,0.013424000392357508
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,7,0.012831999609867731
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,7,0.01321600005030632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,15,0.013050666699806849
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,15,0.01358933374285698
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,31,0.012698666503032049
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,31,0.013466666142145792
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,63,0.012997332960367203
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,63,0.013269333789745966
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,127,0.013301332791646322
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,127,0.014671999961137772
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,255,0.014554666976133982
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,255,0.015578666081031164
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,511,0.016442666451136272
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,511,0.01811733345190684
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,1023,0.020479999482631683
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,1023,0.023024000227451324
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,2047,0.033226666351159416
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,2047,0.03292266776164373
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,4095,0.0583840012550354
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,4095,0.054842665791511536
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,8191,0.09251733620961507
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,8191,0.10113599896430969
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,16383,0.2017973264058431
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,16383,0.18659200270970663
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,32767,0.2972960074742635
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,1,0.01403733342885971
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,1,0.01370666672786077
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,32767,0.35792001088460285
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,3,0.012757333616415659
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,7,0.012960000584522883
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,65535,0.6885813077290853
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,3,0.013914667069911957
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,7,0.013872000078360239
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,15,0.013125333935022354
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,65535,0.6617386738459269
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,15,0.013786666095256805
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,31,0.013253333667914072
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,31,0.013722666849692663
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,63,0.01332266628742218
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,63,0.014240000396966934
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,127,0.013461332768201828
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,255,0.01492799942692121
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,255,0.01611199975013733
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,127,0.01553600033124288
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,511,0.016986666868130367
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,1023,0.02387733260790507
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,511,0.018810667097568512
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,1023,0.02924266705910365
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,2047,0.04432533184687296
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,4095,0.07563733557860057
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,2047,0.03844266633192698
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,4095,0.059690664211908974
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,8191,0.13525333007176718
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,8191,0.10215466221173604
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,1,0.010640000303586325
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,16383,0.1869600017865499
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,16383,0.25525333484013873
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,1,0.011866666376590729
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,3,0.010618666807810465
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,3,0.011711999773979187
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,7,0.012042666474978128
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,32767,0.3596533139546712
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,7,0.010741333166758219
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,15,0.010826667149861654
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,15,0.012047999848922094
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,32767,0.4969013532002767
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,31,0.010645333677530289
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,65535,0.6909226576487223
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,63,0.01055466632048289
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,31,0.011754666765530905
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,63,0.01181866725285848
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,127,0.010735999792814255
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,255,0.013295999417702356
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,65535,1.0767040252685547
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,511,0.014368000129858652
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,127,0.014117332796255747
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,511,0.015530666957298914
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,1023,0.015775999675194424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,255,0.014261333892742792
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,4095,0.023818666736284893
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,1023,0.01661866654952367
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,2047,0.016842667013406754
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,4095,0.021327999730904896
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,2047,0.019461333751678467
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,8191,0.03017599880695343
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,8191,0.033946665624777474
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,16383,0.04823466638724009
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,16383,0.05323199927806854
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,32767,0.08477866649627686
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,32767,0.09264533718427022
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,65535,0.1524853308995565
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,65535,0.18639467159907022
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,3,0.011178666104873022
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,131071,0.2935093243916829
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,131071,0.35438934961954754
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,1,0.010826667149861654
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,1,0.011930666863918304
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,15,0.011973333855470022
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,3,0.01180800050497055
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,7,0.011002667248249054
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,7,0.01227733368674914
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,15,0.010981333752473196
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,31,0.010682666053374609
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,31,0.01190399999419848
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,63,0.010885333021481832
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,63,0.011962667107582092
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,127,0.01099733387430509
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,127,0.014442666123310724
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,255,0.013760000467300415
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,255,0.015200000256299973
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,511,0.014864000181357065
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,511,0.01651200031240781
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,1023,0.01664000004529953
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,1023,0.017456000049908955
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,4095,0.024490666886170704
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,2047,0.017594666530688603
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,2047,0.019882666567961376
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,16383,0.054048001766204834
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,4095,0.021946666141351063
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,8191,0.031141333281993866
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,32767,0.10167466600735982
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,8191,0.03461333364248276
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,16383,0.050704002380371094
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,32767,0.08513599634170532
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,65535,0.15677332878112793
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,65535,0.18519999583562216
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,131071,0.29471466938654584
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,131071,0.35517334938049316
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,1,0.014250667144854864
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,1,0.015461333096027374
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,3,0.014192000031471252
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,7,0.014479999740918478
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,3,0.015402667224407196
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,7,0.015184000134468079
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,15,0.014501333236694336
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,15,0.015200000256299973
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,31,0.014261333892742792
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,31,0.015173333386580149
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,63,0.014549333602190018
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,63,0.015168000012636185
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,127,0.01434133326013883
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,127,0.016997333616018295
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,255,0.015882667154073715
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,255,0.017605333278576534
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,511,0.017680000513792038
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,511,0.02057066683967908
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,1023,0.03027733415365219
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,1023,0.025093334416548412
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,2047,0.04576533536116282
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,2047,0.03947199881076813
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,8191,0.13682132959365845
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,4095,0.07702933251857758
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,4095,0.06135466694831848
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,8191,0.10460266470909119
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,16383,0.2577280004819234
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,16383,0.19046932458877563
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,1,0.01800000046690305
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,3,0.019002666076024372
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,7,0.01759999990463257
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,32767,0.5002400080362955
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,32767,0.35869868596394855
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,15,0.01893866683046023
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,15,0.018042666216691334
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,7,0.0191040001809597
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,31,0.01766933376590411
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,1,0.019189332922299702
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,3,0.017829333742459614
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,63,0.019152000546455383
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,31,0.018986667195955913
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,127,0.018357332795858383
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,127,0.02189333240191142
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,255,0.02162666618824005
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,63,0.017909333109855652
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,255,0.023621333142121632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,511,0.03203733265399933
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,511,0.028666667640209198
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,1023,0.047824000318845115
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,1023,0.04330133398373922
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,4095,0.13555199901262918
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,4095,0.10662933190663655
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,2047,0.07749333480993907
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,2047,0.06553600231806438
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,8191,0.2561226685841878
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,8191,0.193338672320048
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,16383,0.49593599637349445
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,3,0.01961600035429001
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,32767,1.0480960210164387
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,fp8,3,0.020992000897725422
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,7,0.01937599976857503
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,1,0.019845332950353622
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,16383,0.36210131645202637
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,fp8,1,0.021055998901526134
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,32767,0.6983679930369059
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,fp8,7,0.020917333662509918
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,fp8,15,0.02120000123977661
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,31,0.019695999721686046
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,fp8,31,0.021104000508785248
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,15,0.020026666422684986
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,fp8,63,0.020960000654061634
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,fp8,127,0.02418133368094762
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,255,0.02362666775782903
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,fp8,255,0.025407999753952026
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,63,0.01972266659140587
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,511,0.03551999976237615
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,fp8,511,0.03102933367093404
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,1023,0.05136533578236898
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,fp8,1023,0.04614933331807455
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,127,0.02006400004029274
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,fp8,2047,0.06801066795984904
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,8191,0.26100265979766846
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,2047,0.08052800099054973
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,fp8,8191,0.19668267170588175
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,1,0.02611733227968216
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,3,0.025909334421157837
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,fp8,4095,0.1104800005753835
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,7,0.02604266752799352
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,fp8,1,0.027637332677841187
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,fp8,7,0.027722666660944622
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,fp8,3,0.02792000025510788
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,15,0.0262719988822937
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,4095,0.14035733540852866
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,31,0.026517334083716076
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,fp8,31,0.027669332921504974
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,63,0.026421333352724712
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,fp8,63,0.028021333118279774
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,fp8,15,0.02845333268245061
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,127,0.02696000039577484
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,fp8,255,0.03602133442958196
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,255,0.04065600037574768
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,511,0.05571199953556061
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,fp8,511,0.05340800185998281
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,fp8,1023,0.07397866745789845
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,1023,0.08326933284600575
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,fp8,127,0.032672000428040825
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,4095,0.2590773304303487
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,fp8,4095,0.2004800041516622
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,fp8,2047,0.11657599608103435
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,8191,0.49513598283131915
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,2047,0.14134933551152548
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,1,0.03139200061559677
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,fp8,1,0.03278933217128118
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,3,0.03107733279466629
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,fp8,8191,0.3702293237050374
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,fp8,3,0.03288000077009201
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,fp8,15,0.03290133426586787
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,31,0.031397332747777305
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,fp8,31,0.032858667274316154
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,fp8,63,0.03297599901755651
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,fp8,7,0.03313066562016805
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,63,0.0314026673634847
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,7,0.030896000564098358
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,127,0.032111999889214836
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,fp8,127,0.037402667105197906
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,15,0.03149333347876867
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,255,0.04558933277924856
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,fp8,255,0.04178666571776072
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,511,0.06042666733264923
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,fp8,511,0.0588319996992747
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,1023,0.08933867017428081
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,1,0.04310933252175649
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,fp8,1023,0.07974400122960408
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,fp8,1,0.045466666420300804
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,3,0.04264533519744873
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,fp8,3,0.04554133117198944
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,2047,0.14916800459225973
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,fp8,7,0.045552000403404236
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,15,0.04308266441027323
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,fp8,15,0.04568000137805939
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,7,0.04289066791534424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,fp8,2047,0.12325333555539449
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,fp8,31,0.04553066690762838
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,31,0.043418665726979576
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,63,0.04381866753101349
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,fp8,63,0.04553066690762838
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,fp8,4095,0.20780267318089804
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,4095,0.2678346633911133
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,127,0.05095999936262766
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,255,0.06702933212121327
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,fp8,255,0.06897066533565521
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,511,0.09524266918500264
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,fp8,127,0.0547626664241155
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,fp8,511,0.09311999877293904
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,1,0.010954666882753372
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,3,0.011130666981140772
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,3,0.012122667084137598
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,7,0.01097600037852923
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,fp8,1023,0.13251733779907227
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,4095,0.5009546677271525
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,1,0.011839999506870905
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,7,0.01211200033624967
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,1023,0.1519306699434916
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,31,0.01089599976936976
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,31,0.01227733368674914
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,63,0.012154666086037954
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,127,0.011168000598748526
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,127,0.01314666618903478
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,255,0.012351999680201212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,255,0.015487999965747198
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,fp8,4095,0.3853386640548706
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,511,0.015119999647140503
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,511,0.016677333662907284
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,1023,0.01929066702723503
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,15,0.012074666718641916
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,2047,0.021327999730904896
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,2047,0.024304000039895374
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,1023,0.01710933322707812
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,4095,0.03388266762097677
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,4095,0.030218665798505146
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,8191,0.04863999783992767
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,8191,0.05338666836420695
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,16383,0.08458667000134786
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,fp8,2047,0.2185866634051005
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,16383,0.09270933270454407
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,2047,0.2679413358370463
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,32767,0.1543839971224467
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,32767,0.18475733200709024
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,65535,0.35412267843882245
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,1,0.011194666226704916
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,131071,0.6903519630432129
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,1,0.012351999680201212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,3,0.011157333850860596
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,3,0.012047999848922094
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,15,0.011109333485364914
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,63,0.011007999380429586
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,7,0.011370666325092316
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,7,0.012223999947309494
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,15,0.010970667004585266
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,15,0.012309333930412928
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,31,0.011274666835864386
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,31,0.012560000022252401
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,63,0.011285333583752314
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,63,0.012165332833925882
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,127,0.01126933346192042
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,127,0.01333333303531011
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,255,0.012752000242471695
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,255,0.016549333930015564
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,511,0.015557333827018738
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,511,0.01717866708834966
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,1023,0.019920000185569126
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,2047,0.021984001000722248
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,1023,0.01741333305835724
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,2047,0.0244159996509552
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,4095,0.030896000564098358
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,4095,0.03463999927043915
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,8191,0.05514133473237356
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,8191,0.05087466537952423
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,16383,0.10108799735705058
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,16383,0.08560533324877422
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,32767,0.18473599354426065
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,32767,0.15628266334533691
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,65535,0.2916160027186076
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,65535,0.297541340192159
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,131071,0.5729066530863444
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,65535,0.35236799716949463
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,131071,0.5777013301849365
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,131071,0.6881173451741537
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,fp8,1,0.03882133215665817
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,3,0.0383146678407987
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,1,0.038058665891488395
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,fp8,3,0.03923733284076055
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,7,0.038218667109807335
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,fp8,7,0.038746667404969536
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,15,0.03813866774241129
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,fp8,15,0.038736000657081604
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,fp8,31,0.03881066789229711
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,31,0.038021333515644073
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,63,0.03822399924198786
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,fp8,63,0.03909866760174433
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,127,0.043882668018341064
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,fp8,127,0.04736533264319102
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,fp8,255,0.06224533418814341
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,255,0.056757330894470215
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,511,0.08373333017031352
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,fp8,511,0.08566400408744812
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,1023,0.14151466886202493
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,1,0.06418666740258534
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,fp8,1023,0.12586667140324911
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,fp8,1,0.06326933205127716
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,3,0.06485333542029063
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,fp8,3,0.06260799864927928
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,fp8,7,0.06317333380381267
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,2047,0.260042667388916
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,fp8,2047,0.2144213318824768
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,7,0.06520533561706543
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,fp8,15,0.06364800035953522
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,15,0.06488533318042755
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,31,0.06533333162466685
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,fp8,31,0.06306666632493337
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,63,0.07023466626803081
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,fp8,127,0.08710400263468425
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,255,0.09486400087674458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,fp8,255,0.10443733135859172
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,511,0.14756266276041666
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,fp8,511,0.1500746707121531
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,fp8,63,0.06358933448791504
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,127,0.07322666545708974
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,1023,0.2617759903271993
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,1,0.0651146670182546
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,fp8,1,0.06543466448783875
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,3,0.06457599997520447
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,fp8,1023,0.2316746711730957
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,fp8,3,0.06577066580454509
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,fp8,7,0.0654666672150294
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,15,0.06463466584682465
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,fp8,15,0.06591466565926869
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,fp8,31,0.06618666648864746
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,2047,0.4983946482340495
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,7,0.06492266555627187
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,fp8,2047,0.4106186628341675
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,31,0.06611733138561249
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,63,0.07151466608047485
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,fp8,63,0.06654400130112965
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,127,0.073594664533933
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,fp8,127,0.08892266949017842
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,fp8,511,0.1574613352616628
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,511,0.15061333775520325
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,fp8,1,0.11340266466140747
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,1,0.11627733707427979
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,7,0.1160533328851064
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,1023,0.2648640076319377
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,3,0.11626133322715759
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,fp8,3,0.11355200409889221
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,fp8,255,0.11077333490053813
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,255,0.09602133433024089
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,fp8,1023,0.23437867561976114
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,fp8,7,0.11340266466140747
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,15,0.11738133430480957
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,31,0.12596266468365988
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,fp8,31,0.11502400040626526
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,fp8,63,0.12457066774368286
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,fp8,127,0.15808533628781637
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,63,0.12706666191418967
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,255,0.17343467473983765
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,fp8,15,0.11275200049082439
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,1,0.012144000579913458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,3,0.011231999844312668
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,3,0.012266666938861212
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,7,0.011125333607196808
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,7,0.012256000190973282
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,15,0.011407999942700068
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,15,0.0124746672809124
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,31,0.011258666714032492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,fp8,255,0.1906506617863973
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,31,0.0122079998254776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,1,0.01098666712641716
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,127,0.12964266538619995
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,63,0.011258666714032492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,127,0.011258666714032492
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,127,0.013397333522637686
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,255,0.014122666170199713
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,511,0.014682666709025701
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,511,0.017103999853134155
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,1023,0.018895999838908512
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,1023,0.02160533269246419
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,2047,0.03166399896144867
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,2047,0.027450665831565857
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,4095,0.05060799916585287
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,4095,0.04535466432571411
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,255,0.012624000509579977
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,16383,0.1508853336175283
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,8191,0.08162666857242584
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,8191,0.09368000427881877
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,16383,0.18239466349283853
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,32767,0.2886613408724467
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,32767,0.3524746497472127
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,65535,0.5654133160909017
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,65535,0.6814186573028564
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,63,0.012634667257467905
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,1,0.011695999652147293
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,1,0.012768000364303589
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,131071,1.1270453135172527
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,3,0.011839999506870905
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,131071,1.3467733065287273
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,3,0.012928000340859095
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,7,0.011589333415031433
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,7,0.0129120002190272
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,15,0.012821332861979803
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,15,0.01163200040658315
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,31,0.011727999895811081
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,511,0.27516265710194904
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,31,0.012949333836634954
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,63,0.011930666863918304
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,63,0.012847999731699625
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,127,0.011989332735538483
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,127,0.014639999717473984
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,255,0.014981333166360855
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,511,0.015306666493415833
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,511,0.01762666677435239
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,1023,0.019621333728233974
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,1023,0.022533332308133442
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,fp8,511,0.28471465905507404
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,2047,0.027893332143624622
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,2047,0.032560000816980995
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,4095,0.04750399788220724
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,4095,0.05209066470464071
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,8191,0.0831413318713506
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,16383,0.18203733364741007
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,16383,0.15387200315793356
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,32767,0.2948533296585083
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,32767,0.34827200571695965
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,65535,0.571125348409017
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,65535,0.6849973201751709
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,1,0.011637333780527115
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,1,0.012693333129088083
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,131071,1.1400480270385742
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,131071,1.3427626291910808
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,3,0.011839999506870905
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,3,0.012576000144084295
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,7,0.011706666400035223
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,15,0.011882666498422623
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,7,0.01257066677014033
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,15,0.01267733300725619
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,255,0.013631999492645264
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,31,0.01156266654531161
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,31,0.012597333639860153
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,8191,0.09909333785374959
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,63,0.011941333611806234
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,63,0.01333333303531011
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,127,0.012069333344697952
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,127,0.014266667266686758
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,255,0.013658666362365087
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,255,0.015135999768972397
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,511,0.014869333555301031
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,511,0.01724799970785777
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,1023,0.019834666202465694
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,1023,0.022304000953833263
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,2047,0.028394666810830433
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,2047,0.03206400076548258
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,4095,0.04801600178082784
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,4095,0.05249066650867462
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,8191,0.08387733499209087
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,1023,0.5524319807688395
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,8191,0.1002506713072459
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,16383,0.15384533007939658
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,16383,0.18357332547505698
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,fp8,1023,0.44338667392730713
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,32767,0.2924693425496419
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,32767,0.35205864906311035
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,65535,0.5772586663564047
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,3,0.011168000598748526
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,1,0.011440000186363855
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,7,0.01116266722480456
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,131071,1.1375412940979004
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,7,0.012080000092585882
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,3,0.012005332857370377
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,15,0.011002667248249054
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,131071,1.3496747016906738
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,15,0.01240533341964086
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,31,0.012250666817029318
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,63,0.011125333607196808
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,31,0.011168000598748526
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,63,0.012026666353146235
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,127,0.014709333578745524
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,255,0.013482666263977686
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,127,0.011312000453472137
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,255,0.014688000082969666
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,511,0.013781332721312841
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,511,0.014576000471909841
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,1023,0.01381333296497663
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,1023,0.014602666099866232
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,2047,0.014938666174809137
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,2047,0.015829333414634068
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,4095,0.01603200038274129
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,4095,0.016783999900023144
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,8191,0.018053332964579265
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,8191,0.019402666638294857
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,16383,0.022682666778564453
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,16383,0.0249493345618248
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,32767,0.032586666444937386
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,32767,0.03534399966398875
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,65535,0.050704002380371094
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,65535,0.05579733351866404
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,131071,0.08919999996821086
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,131071,0.09864532947540283
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,1,0.01118933285276095
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,1,0.012437333663304647
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,3,0.011029332876205444
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,3,0.0122079998254776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,7,0.010863999525705973
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,7,0.012282667060693106
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,15,0.011242666592200598
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,15,0.012053333222866058
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,31,0.011077333241701126
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,31,0.011999999483426413
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,63,0.011157333850860596
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,63,0.012357333054145178
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,65535,0.690666675567627
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,1,0.012154666086037954
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,127,0.011215999722480774
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,127,0.014629332969586054
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,255,0.013647999614477158
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,1023,0.01598400001724561
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,255,0.014602666099866232
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,2047,0.016735999534527462
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,511,0.013717333475748697
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,511,0.014815999815861383
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,1023,0.014991999914248785
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,2047,0.0161013330022494
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,4095,0.018218666315078735
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,4095,0.019733333339293797
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,8191,0.021536000072956085
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,8191,0.024282666544119518
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,16383,0.030095999439557392
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,16383,0.034143999218940735
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,32767,0.047653332352638245
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,32767,0.05340800185998281
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,65535,0.08434133728345235
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,65535,0.09331732988357544
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,131071,0.15380799770355225
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,131071,0.18465065956115723
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,1,0.012949333836634954
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,1,0.013765333841244379
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,3,0.012650666137536367
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,3,0.013786666095256805
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,7,0.012874666601419449
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,7,0.013872000078360239
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,15,0.013568000247081121
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,15,0.01268799975514412
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,31,0.01301866645614306
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,31,0.013248000293970108
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,63,0.012986666212479273
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,127,0.014629332969586054
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,63,0.013765333841244379
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,127,0.013253333667914072
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,255,0.014469332993030548
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,255,0.015930666277805965
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,511,0.016330666840076447
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,511,0.01829333355029424
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,1023,0.020655999581019085
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,1023,0.023205332458019257
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,2047,0.034058667719364166
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,2047,0.03310933212439219
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,4095,0.05449600021044413
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,4095,0.0583840012550354
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,8191,0.09532266855239868
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,8191,0.10076799988746643
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,16383,0.15692800283432007
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,16383,0.18630399306615195
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,1,0.010591999938090643
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,1,0.012144000579913458
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,32767,0.35611732800801593
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,3,0.010608000059922537
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,3,0.012085333466529846
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,32767,0.29877867301305133
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,7,0.01190399999419848
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,7,0.01108266661564509
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,65535,0.7877066930135092
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,65535,0.6929439703623453
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,15,0.011391999820868174
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,31,0.010677333921194077
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,31,0.01173866664369901
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,15,0.011717333147923151
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,131071,1.3647947311401367
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,63,0.010640000303586325
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,127,0.011109333485364914
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,127,0.014357333381970724
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,63,0.01184533288081487
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,255,0.013327999661366144
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,131071,1.3006772994995117
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,255,0.014432000617186228
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,511,0.01532799998919169
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,511,0.014560000350077948
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,1023,0.015813333292802174
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,1023,0.01655999943614006
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,2047,0.016842667013406754
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,2047,0.019359999646743137
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,4095,0.023685333629449207
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,8191,0.029861333469549816
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,4095,0.02146133283774058
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,8191,0.03379733363787333
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,16383,0.04841599861780802
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,16383,0.052943999568621315
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,32767,0.08502399921417236
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,32767,0.09346133470535278
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,65535,0.15398933490117392
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,65535,0.18639467159907022
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,131071,0.29126399755477905
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,131071,0.35339732964833576
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,1,0.014202666779359182
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,1,0.01533866673707962
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,3,0.014085333794355392
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,3,0.015477333217859268
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,7,0.014069333672523499
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,7,0.015263999501864115
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,15,0.013925333817799887
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,15,0.015311999867359797
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,31,0.01421333352724711
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,31,0.015290666371583939
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,63,0.014282666146755219
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,63,0.015290666371583939
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,127,0.014314666390419006
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,127,0.016528000434239704
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,255,0.015941333025693893
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,255,0.017829333742459614
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,511,0.01854933301607768
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,511,0.020319999506076176
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,1023,0.030415999392668407
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,1023,0.025258667767047882
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,2047,0.04565866788228353
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,2047,0.0401706670721372
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,8191,0.1402666668097178
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,16383,0.25809599955876666
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,8191,0.10569600264231364
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,16383,0.19031999508539835
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,4095,0.0759093314409256
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,4095,0.06043200194835663
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,1,0.019909333437681198
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,32767,0.36454399426778156
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,1,0.020901332298914593
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,32767,0.5012693405151367
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,65535,0.6990133126576742
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,7,0.019519999623298645
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,7,0.02096533278624217
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,3,0.021040000021457672
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,3,0.01966933285196622
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,65535,1.0742560227711995
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,31,0.019920000185569126
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,31,0.021338666478792827
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,15,0.019567999988794327
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,63,0.02015999952952067
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,15,0.021130666136741638
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,63,0.02130666623512904
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,127,0.02380799998839696
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,255,0.02332266668478648
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,255,0.025616000096003216
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,511,0.03595733394225439
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,511,0.031354665756225586
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,1023,0.05117333432038625
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,1023,0.045226668318112694
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,127,0.02004266654451688
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,2047,0.08178133269151051
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,2047,0.06843733290831248
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,8191,0.19629865884780884
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,4095,0.14065600434939066
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,8191,0.25885866085688275
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,16383,0.4939200083414714
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,16383,0.36691733201344806
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,fp8,1,0.03260799994071325
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,1,0.031130666534105938
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,4095,0.1102186640103658
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,3,0.030896000564098358
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,7,0.031285333136717476
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,fp8,15,0.032858667274316154
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,63,0.031199999153614044
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,31,0.0314026673634847
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,fp8,3,0.03264000018437704
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,fp8,63,0.03311466674009959
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,fp8,7,0.03276800115903219
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,15,0.03121600051720937
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,127,0.03145066648721695
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,fp8,127,0.03668266783157984
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,255,0.04523199796676636
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,fp8,255,0.04167466859022776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,511,0.06039999922116598
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,fp8,511,0.05877333382765452
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,fp8,31,0.03283733377854029
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,fp8,1023,0.07955733438332875
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,1023,0.08956799904505412
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,1,0.01190399999419848
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,3,0.011034666250149408
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,2047,0.1479466656843821
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,3,0.011792000383138657
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,fp8,2047,0.1236853301525116
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,fp8,8191,0.3790666659673055
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,1,0.011077333241701126
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,7,0.011968000481526056
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,8191,0.5002133448918661
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,31,0.011087999989589056
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,31,0.011760000139474869
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,63,0.012149333953857422
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,127,0.01099733387430509
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,15,0.01219733307758967
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,127,0.013253333667914072
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,255,0.012069333344697952
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,255,0.015664000064134598
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,511,0.015008000036080679
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,511,0.016421332955360413
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,1023,0.017237332959969837
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,63,0.01108266661564509
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,1023,0.019237333287795384
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,2047,0.02130666623512904
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,2047,0.023898666103680927
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,4095,0.031504000226656594
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,4095,0.033930666744709015
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,8191,0.047594666481018066
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,8191,0.053301334381103516
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,16383,0.0851039985815684
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,16383,0.0932426651318868
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,32767,0.15380266308784485
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,4095,0.26628265778223675
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,7,0.01108266661564509
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,15,0.01098666712641716
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,32767,0.18555732568105063
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,fp8,4095,0.20690667629241943
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,65535,0.294378658135732
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,65535,0.3532426754633586
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,131071,0.572165330251058
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,131071,0.6878346602121989
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,1,0.037818667789300285
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,fp8,1,0.03874133278926214
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,3,0.03825066735347112
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,fp8,7,0.03871466716130575
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,15,0.03800000001986822
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,7,0.03845333307981491
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,fp8,15,0.03882666677236557
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,fp8,3,0.038506666819254555
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,fp8,31,0.03885866701602936
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,63,0.03917866696914037
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,fp8,63,0.03868266691764196
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,127,0.043893332282702126
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,fp8,127,0.04833066463470459
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,255,0.05606399973233541
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,fp8,255,0.061978667974472046
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,511,0.08358933528264363
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,fp8,511,0.0849120020866394
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,1023,0.14130666851997375
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,fp8,1023,0.1262933313846588
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,31,0.03818666686614355
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,2047,0.25920534133911133
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,fp8,2047,0.21338667472203574
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,3,0.06458133459091187
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,fp8,3,0.06598400076230367
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,1,0.06517333288987477
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,fp8,1,0.06600533425807953
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,4095,0.49505066871643066
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,7,0.06489066779613495
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,fp8,4095,0.38078399499257404
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,15,0.06477866570154826
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,fp8,15,0.06578666468461354
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,fp8,7,0.06571733454863231
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,31,0.06570133566856384
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,fp8,31,0.06612800061702728
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,fp8,63,0.06664533416430156
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,fp8,127,0.08847467104593913
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,255,0.09520000219345093
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,fp8,255,0.10983467102050781
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,511,0.14915733536084494
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,fp8,511,0.15738133589426676
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,1,0.011317333827416102
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,1,0.012389333297808966
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,3,0.011173332730929056
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,3,0.012181332955757776
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,7,0.011146667102972666
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,7,0.012367999802033106
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,15,0.011317333827416102
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,15,0.012309333930412928
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,31,0.011285333583752314
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,31,0.01257066677014033
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,63,0.011152000476916632
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,63,0.012629333883523941
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,127,0.011247999966144562
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,127,0.013594667116800943
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,255,0.013023999830087027
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,255,0.01434133326013883
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,511,0.014533333480358124
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,511,0.01718933383623759
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,1023,0.018965333700180054
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,1023,0.021712000171343487
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,2047,0.02757866680622101
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,2047,0.03177600105603536
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,4095,0.045498669147491455
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,4095,0.05107733110586802
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,127,0.0737013320128123
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,63,0.07109333574771881
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,1023,0.26495999097824097
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,8191,0.08253333469231923
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,8191,0.09250666697820027
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,fp8,1023,0.23560533920923868
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,32767,0.2922026713689168
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,16383,0.1840533415476481
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,16383,0.15198933084805807
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,65535,0.569978674252828
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,131071,1.1268213589986165
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,131071,1.3657066027323406
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,32767,0.35043732325236004
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,65535,0.6860586802164713
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,fp8,2047,0.41044799486796063
VLLM,0.12.0,NVIDIA H200,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,2047,0.5337920188903809
