framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,beam_width,attn_dtype,kv_cache_dtype,step,latency
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,1,0.01210133358836174
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,31,0.01219733307758967
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,fp8,1,0.01357866699496905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,fp8,31,0.013023999830087027
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,fp8,63,0.013013333082199097
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,63,0.012128000458081564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,127,0.012319999436537424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,fp8,127,0.014794666320085526
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,255,0.01370666672786077
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,fp8,255,0.015696000307798386
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,511,0.015397333850463232
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,fp8,511,0.01791999985774358
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,1023,0.020351999749739964
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,fp8,1023,0.022410665949185688
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,2047,0.02921066681543986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,fp8,2047,0.03229333211978277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,4095,0.04844800134499868
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,fp8,4095,0.054085334142049156
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,1,0.01250133290886879
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,fp8,1,0.01328533391157786
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,3,0.012144000579913458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,3,0.012245333443085352
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,fp8,3,0.013295999417702356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,7,0.01228800043463707
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,fp8,7,0.013264000415802002
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,15,0.012175999581813812
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,fp8,15,0.01332266628742218
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,31,0.01246400053302447
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,fp8,31,0.013493333011865616
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,63,0.012053333222866058
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,fp8,63,0.013306666165590286
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,127,0.012357333054145178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,fp8,127,0.01458666721979777
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,7,0.012074666718641916
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,255,0.014170666535695394
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,fp8,255,0.01553600033124288
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,511,0.01647466669480006
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,fp8,3,0.013290667285521826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,fp8,511,0.017887999614079792
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,1023,0.024501333634058636
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,fp8,1023,0.02276266614596049
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,2047,0.037477334340413414
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,fp8,2047,0.03490666548411051
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,4095,0.061247999469439186
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,fp8,4095,0.057775999108950295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,1,0.012554666648308435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,fp8,1,0.01392000044385592
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,fp8,3,0.013744000345468521
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,3,0.012629333883523941
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,fp8,7,0.014117332796255747
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,7,0.012522666404644648
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,15,0.012837332983811697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,fp8,15,0.013733333597580591
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,31,0.012730666746695837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,fp8,31,0.014085333794355392
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,fp8,63,0.01404800017674764
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,63,0.012831999609867731
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,127,0.012879999975363413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,fp8,127,0.015482666591803232
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,255,0.014970666418472925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,fp8,255,0.016095999628305435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,fp8,15,0.013354666531085968
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,511,0.01740266631046931
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,fp8,511,0.018511999398469925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,15,0.012266666938861212
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,1023,0.03621866554021835
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,fp8,7,0.013823999712864557
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,fp8,1023,0.024298667907714844
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,2047,0.057722667853037514
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,fp8,2047,0.04190933207670847
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,fp8,1,0.01754133279124896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,4095,0.10123200217882793
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,fp8,3,0.017690667261679966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,fp8,4095,0.06613866488138835
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,1,0.016469333320856094
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,3,0.016730666160583496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,7,0.016693333784739178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,fp8,7,0.018005333840847015
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,15,0.016586666305859882
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,fp8,15,0.017562666287024815
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,31,0.016623999923467636
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,fp8,31,0.01741333305835724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,63,0.016778666526079178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,fp8,63,0.017887999614079792
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,127,0.017632000148296356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,fp8,127,0.0207893339296182
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,255,0.021962667504946392
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,fp8,255,0.02199999988079071
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,511,0.037834666669368744
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,fp8,511,0.027930667002995808
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,1023,0.06084799766540527
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,fp8,1023,0.04417066772778829
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,2047,0.10470933715502422
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,fp8,2047,0.0693333347638448
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,4095,0.1899413267771403
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,fp8,4095,0.11594133575757344
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,1,0.011605333536863327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,3,0.011514666179815928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,1,0.012421333541472753
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,3,0.012757333616415659
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,7,0.011605333536863327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,7,0.012624000509579977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,15,0.011600000162919363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,15,0.012565333396196365
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,31,0.011589333415031433
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,127,0.01145600030819575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,31,0.012847999731699625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,63,0.011391999820868174
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,63,0.012618667135636011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,127,0.014101333916187286
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,255,0.012885333349307379
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,255,0.014186666657527288
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,511,0.013199999928474426
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,511,0.014282666146755219
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,1023,0.01333333303531011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,1023,0.014570667097965876
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,2047,0.014831999937693277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,2047,0.01570133368174235
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,4095,0.016058667252461117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,4095,0.017263999829689663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,8191,0.018160000443458557
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,8191,0.019589333484570186
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,16383,0.022757334013779957
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,16383,0.025472000241279602
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,1,0.01145600030819575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,1,0.012970666090647379
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,3,0.011461333682139715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,15,0.012618667135636011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,3,0.012794667234023413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,7,0.011349332829316458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,7,0.0124746672809124
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,15,0.01157333329319954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,31,0.011727999895811081
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,31,0.012752000242471695
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,63,0.011472000430027643
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,63,0.012885333349307379
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,127,0.011413333316644033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,127,0.013493333011865616
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,255,0.012794667234023413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,255,0.013776000589132309
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,511,0.012954667210578918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,511,0.014186666657527288
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,1023,0.01360000049074491
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,1023,0.014954666296641031
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,2047,0.014645333091417948
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,8191,0.018981333822011948
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,2047,0.016575999557971954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,4095,0.018090666582187016
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,4095,0.017557332913080852
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,8191,0.02042666698495547
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,16383,0.023472001155217487
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,16383,0.025861332813898723
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,1,0.01181866725285848
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,1,0.012613333761692047
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,3,0.011711999773979187
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,3,0.013023999830087027
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,7,0.011936000237862269
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,7,0.012831999609867731
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,15,0.01157333329319954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,15,0.012890666723251343
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,31,0.011866666376590729
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,31,0.01268799975514412
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,63,0.01173866664369901
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,63,0.012986666212479273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,127,0.011871999750534693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,127,0.014005333185195923
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,255,0.012639999389648438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,255,0.014303999642531076
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,511,0.013317332913478216
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,511,0.014794666320085526
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,1023,0.014282666146755219
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,1023,0.01562133307258288
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,2047,0.01540800059835116
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,2047,0.016837333639462788
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,4095,0.016517333686351776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,4095,0.018272000054518383
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,8191,0.01882133384545644
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,8191,0.02021866664290428
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,16383,0.02478933334350586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,16383,0.02700799951950709
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,1,0.011973333855470022
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,1,0.012453333785136541
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,15,0.011930666863918304
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,3,0.01163200040658315
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,3,0.012773333738247553
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,7,0.011834666132926941
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,7,0.012389333297808966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,15,0.012650666137536367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,31,0.011920000116030375
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,31,0.012538666526476542
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,63,0.011941333611806234
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,63,0.012383999923865
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,127,0.012149333953857422
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,127,0.013503999759753546
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,255,0.013232000172138214
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,255,0.014917333920796713
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,511,0.013983999689420065
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,511,0.015178666760524115
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,1023,0.014655999839305878
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,1023,0.015189333508412043
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,2047,0.016165333489576977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,2047,0.016565332810084026
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,4095,0.016879999389251072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,4095,0.017802666872739792
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,8191,0.020629333953062694
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,8191,0.02056533346573512
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,16383,0.03798400113979975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,16383,0.026842666169007618
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,1,0.011546666423479715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,1,0.012586666891972223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,3,0.011477333803971609
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,3,0.012879999975363413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,7,0.011429333438475927
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,7,0.012901333471139273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,15,0.011594666788975397
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,15,0.012624000509579977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,31,0.011472000430027643
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,31,0.01313599944114685
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,63,0.011514666179815928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,63,0.013151999562978745
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,127,0.011546666423479715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,127,0.014453332871198654
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,255,0.013077333569526672
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,255,0.014010666559139887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,511,0.013194666554530462
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,511,0.014645333091417948
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,1023,0.014378666877746582
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,1023,0.01545599972208341
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,2047,0.015791999797026317
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,2047,0.01666133354107539
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,4095,0.017781333376963932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,4095,0.020138667275508244
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,8191,0.021498667697111767
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,8191,0.024101334313551586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,16383,0.030037333567937214
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,16383,0.034143999218940735
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,1,0.011733333269755045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,1,0.01240533341964086
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,3,0.011642667154471079
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,3,0.012810666114091873
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,7,0.012879999975363413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,7,0.01157333329319954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,15,0.01184533288081487
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,15,0.01257066677014033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,31,0.01145600030819575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,31,0.012997332960367203
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,63,0.011535999675591787
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,63,0.012698666503032049
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,127,0.011765333513418833
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,127,0.013951999445756277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,255,0.012821332861979803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,255,0.013914667069911957
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,511,0.013061333447694778
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,511,0.015013333410024643
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,1023,0.014432000617186228
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,1023,0.016224000602960587
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,2047,0.01659199967980385
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,2047,0.017535999417304993
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,4095,0.01826133330663045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,4095,0.019727999965349834
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,8191,0.021957332889238994
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,8191,0.024346667031447094
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,16383,0.031701333820819855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,16383,0.03444266567627589
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,1,0.011887999872366587
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,1,0.012682666381200155
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,3,0.011802667131026586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,3,0.012698666503032049
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,7,0.01156266654531161
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,7,0.012794667234023413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,15,0.011525332927703857
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,15,0.012874666601419449
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,31,0.011952000359694162
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,31,0.012752000242471695
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,63,0.011930666863918304
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,63,0.012736000120639801
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,127,0.011567999919255575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,127,0.014074667046467463
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,255,0.013141332815090815
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,255,0.014826666563749313
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,511,0.013701333353916803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,511,0.015125333021084467
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,1023,0.01587733378012975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,1023,0.016309333344300587
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,2047,0.016330666840076447
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,2047,0.01749333366751671
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,4095,0.018624000251293182
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,4095,0.020058666666348774
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,8191,0.02258666604757309
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,8191,0.02477866659561793
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,16383,0.040192000567913055
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,16383,0.03716266651948293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,1,0.012165332833925882
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,1,0.012821332861979803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,3,0.012096000214417776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,3,0.012863999853531519
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,7,0.012138667205969492
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,7,0.012730666746695837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,63,0.012223999947309494
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,15,0.012202666451533636
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,15,0.012714666624863943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,31,0.012175999581813812
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,31,0.012831999609867731
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,63,0.01293333371480306
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,127,0.012250666817029318
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,511,0.015290666371583939
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,127,0.014938666174809137
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,1023,0.016208000481128693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,255,0.013669333110253016
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,255,0.014362666755914688
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,511,0.014293332894643148
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,1023,0.016309333344300587
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,2047,0.016735999534527462
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,2047,0.017360000560681026
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,4095,0.020410666863123577
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,4095,0.019946667055288952
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,8191,0.038848000268141426
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,8191,0.024645333488782246
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,16383,0.059450666109720864
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,16383,0.04612799982229868
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,1,0.013365333278973898
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,3,0.01314666618903478
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,fp8,1,0.014576000471909841
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,7,0.013365333278973898
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,fp8,3,0.014741333822409311
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,fp8,7,0.01470400020480156
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,15,0.013455999394257864
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,fp8,15,0.014560000350077948
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,31,0.013306666165590286
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,fp8,31,0.01479999969402949
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,63,0.01328533391157786
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,fp8,63,0.014767999450365702
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,127,0.013888000200192133
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,511,0.01695999999841054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,fp8,127,0.016058667252461117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,255,0.01552533358335495
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,fp8,255,0.016549333930015564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,fp8,511,0.019146667172511418
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,1023,0.023077333966890972
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,fp8,1023,0.0240639994541804
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,fp8,2047,0.03583999971548716
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,1,0.013701333353916803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,2047,0.03737599899371465
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,fp8,1,0.014917333920796713
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,3,0.0136266661187013
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,fp8,3,0.014922666052977243
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,7,0.013557333499193192
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,fp8,7,0.014725333700577417
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,15,0.013477332890033722
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,fp8,15,0.014954666296641031
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,31,0.01368533323208491
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,fp8,31,0.015072000523408255
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,63,0.013663999736309052
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,fp8,63,0.015050667027632395
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,127,0.014106666048367819
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,fp8,127,0.01613866661985715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,255,0.015429332852363586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,fp8,255,0.016970666746298473
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,511,0.018453333526849747
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,fp8,511,0.019658666104078293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,1023,0.036992001036802925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,fp8,1023,0.025253333151340485
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,2047,0.05849599838256836
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,1,0.017071999609470367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,fp8,2047,0.04308799902598063
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,fp8,1,0.019082666685183842
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,3,0.017360000560681026
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,fp8,3,0.019007999449968338
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,7,0.017221332838137943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,fp8,7,0.01883200059334437
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,fp8,15,0.0183999997874101
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,15,0.017360000560681026
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,fp8,31,0.019029332945744198
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,31,0.017701332767804463
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,63,0.01740266631046931
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,fp8,63,0.018863999595244724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,127,0.017674667139848072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,fp8,127,0.02145066608985265
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,255,0.021349333226680756
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,fp8,255,0.022639999787012737
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,511,0.03920533259709676
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,fp8,511,0.0305226668715477
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,2047,0.10432533423105876
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,fp8,2047,0.07108266651630402
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,1023,0.06178133189678192
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,1,0.023978665471076965
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,fp8,1023,0.04622933268547058
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,fp8,1,0.025194667279720306
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,3,0.024271999796231587
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,7,0.024218666056791942
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,fp8,3,0.025616000096003216
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,fp8,7,0.025424001117547352
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,fp8,15,0.025013332565625507
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,15,0.024122667809327442
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,fp8,31,0.02526933451493581
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,63,0.024383999407291412
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,31,0.024383999407291412
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,fp8,63,0.025498665869235992
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,127,0.027189334233601887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,fp8,127,0.030261332790056866
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,255,0.04197333256403605
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,fp8,255,0.03475199888149897
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,511,0.06328533093134563
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,fp8,511,0.05120000243186951
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,1023,0.10593066612879436
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,fp8,1023,0.07488533357779185
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,2047,0.19261866807937622
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,1,0.011109333485364914
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,fp8,2047,0.12148267030715942
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,1,0.011957333733638128
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,3,0.012522666404644648
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,7,0.010863999525705973
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,3,0.011391999820868174
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,7,0.011952000359694162
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,15,0.01126933346192042
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,15,0.011994666109482447
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,31,0.011061333119869232
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,31,0.012645332763592402
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,63,0.010805333654085795
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,63,0.012842666357755661
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,127,0.01139733319481214
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,127,0.013807999591032663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,255,0.012527999778588613
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,255,0.013893333574136099
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,511,0.013866666704416275
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,511,0.015279999623696009
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,1023,0.015200000256299973
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,1023,0.016575999557971954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,2047,0.01666133354107539
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,2047,0.01854933301607768
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,4095,0.02090666691462199
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,4095,0.023365333676338196
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,8191,0.029626667499542236
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,8191,0.033786666889985405
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,16383,0.04845866560935974
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,16383,0.0529013325770696
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,1,0.011237333218256632
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,1,0.012165332833925882
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,3,0.011301333705584208
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,3,0.012282667060693106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,7,0.011546666423479715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,7,0.012671999633312225
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,15,0.011370666325092316
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,15,0.012223999947309494
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,31,0.011205332974592844
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,31,0.012261333564917246
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,63,0.011226666470368704
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,63,0.01303999995191892
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,127,0.011301333705584208
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,127,0.013978666315476099
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,255,0.012906666845083237
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,511,0.013957332819700241
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,255,0.014570667097965876
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,511,0.016154666741689045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,1023,0.015573333948850632
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,1023,0.017173333714405697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,2047,0.017488000293572743
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,2047,0.01923199991385142
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,4095,0.02102400114138921
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,16383,0.0563679983218511
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,4095,0.023999998966852825
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,8191,0.03054933249950409
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,8191,0.034245334565639496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,16383,0.05067733426888784
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,1,0.011695999652147293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,1,0.012842666357755661
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,3,0.011594666788975397
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,3,0.012778667112191519
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,7,0.01146666705608368
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,63,0.011701333026091257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,7,0.012682666381200155
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,15,0.011770666887362799
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,127,0.011685332904259363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,127,0.014298666268587112
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,15,0.012784000486135483
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,31,0.011610666910807291
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,31,0.012800000607967377
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,63,0.012890666723251343
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,255,0.0138026662170887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,255,0.015130666395028433
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,511,0.01482133318980535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,511,0.016261332978804905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,1023,0.01602666700879733
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,1023,0.017370666066805523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,2047,0.017263999829689663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,2047,0.019621333728233974
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,4095,0.02197333425283432
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,4095,0.023941333095232647
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,8191,0.04246933261553446
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,8191,0.03595199932654699
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,3,0.012629333883523941
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,16383,0.07183466851711273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,16383,0.05905599892139435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,1,0.012165332833925882
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,1,0.013616000612576803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,3,0.013072000195582708
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,7,0.012304000556468964
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,7,0.012714666624863943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,15,0.012400000045696894
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,15,0.013130666067202887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,31,0.012821332861979803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,31,0.013434667140245438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,63,0.012752000242471695
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,63,0.013167999684810638
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,127,0.012698666503032049
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,511,0.015381333728631338
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,127,0.014074667046467463
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,255,0.01393066719174385
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,255,0.014805333067973455
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,511,0.016373333831628162
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,1023,0.016757333030303318
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,1023,0.01669866715868314
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,2047,0.01865600049495697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,2047,0.019130667050679524
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,4095,0.03722133239110311
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,4095,0.024682665864626568
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,8191,0.05835733314355215
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,8191,0.046207999189694725
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,16383,0.1025279959042867
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,16383,0.07423999905586243
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,1,0.015775999675194424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,fp8,1,0.017157333592573803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,3,0.016143999993801117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,fp8,3,0.017136000096797943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,7,0.01591466615597407
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,fp8,7,0.016986666868130367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,15,0.016309333344300587
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,fp8,15,0.01676799977819125
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,31,0.015882667154073715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,63,0.016085332880417507
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,fp8,31,0.0173333336909612
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,fp8,63,0.017301333447297413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,127,0.01628799984852473
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,255,0.017829333742459614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,fp8,255,0.01931200052301089
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,511,0.020645332833131153
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,fp8,127,0.019088000059127808
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,fp8,511,0.02180800090233485
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,1023,0.03956799954175949
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,fp8,1023,0.028031999866167705
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,1,0.018895999838908512
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,fp8,1,0.020773333807786305
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,3,0.019167999426523846
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,fp8,3,0.021312000850836437
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,fp8,7,0.020794666061798733
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,fp8,15,0.020597333709398907
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,31,0.018911999960740406
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,7,0.01924266666173935
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,fp8,31,0.0207893339296182
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,15,0.019039999693632126
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,63,0.019280000279347103
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,fp8,63,0.021226666867733
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,127,0.019925333559513092
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,fp8,127,0.02345066765944163
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,255,0.02430933217207591
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,fp8,255,0.02491733431816101
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,511,0.04268266757329305
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,fp8,511,0.032314665615558624
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,1,0.02590399980545044
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,1023,0.06403733293215434
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,fp8,1,0.027994667490323383
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,3,0.025557334224383037
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,fp8,1023,0.050570666790008545
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,15,0.026165333886941273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,7,0.02589333305756251
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,fp8,3,0.027797333896160126
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,fp8,7,0.027727998793125153
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,31,0.025909334421157837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,fp8,15,0.02827200045188268
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,fp8,31,0.027615999182065327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,63,0.026911998788515728
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,fp8,63,0.028207999964555103
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,127,0.02972800036271413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,fp8,127,0.032629333436489105
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,255,0.04563733438650767
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,fp8,255,0.03822399924198786
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,511,0.06774933139483134
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,1,0.038389332592487335
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,fp8,511,0.05499733487764994
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,fp8,1,0.0410453329483668
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,3,0.03822399924198786
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,fp8,3,0.04095999896526337
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,7,0.03892799963553747
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,fp8,7,0.04065066576004028
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,fp8,1023,0.07897600034872691
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,1023,0.11003733674685161
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,15,0.038191998998324074
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,fp8,15,0.04111466556787491
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,31,0.038575999438762665
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,fp8,31,0.04074133435885111
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,63,0.04163199911514918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,fp8,63,0.04065066576004028
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,255,0.06663999954859416
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,1023,0.19801600774129233
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,fp8,1023,0.13294933239618936
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,fp8,255,0.06317333380381267
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,127,0.04814933240413666
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,fp8,127,0.05407466491063436
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,511,0.11081066727638245
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,fp8,511,0.08693333466847737
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,1,0.024362665911515553
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,3,0.02443733314673106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,fp8,1,0.02622399975856145
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,7,0.024671999116738636
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,fp8,15,0.025754667818546295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,fp8,7,0.026122666895389557
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,fp8,3,0.025909334421157837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,15,0.024432001014550526
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,31,0.024533333877722423
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,fp8,31,0.026330667237440746
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,63,0.02475733309984207
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,fp8,63,0.025909334421157837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,127,0.025781333446502686
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,fp8,127,0.028490667541821797
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,255,0.03267733256022135
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,fp8,255,0.03052799900372823
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,1,0.03062933435042699
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,fp8,1,0.0329066663980484
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,3,0.030346666773160298
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,fp8,3,0.03299733251333237
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,7,0.03057066599527995
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,fp8,7,0.0325546662012736
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,15,0.031013332307338715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,fp8,15,0.03306666761636734
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,31,0.0312266672650973
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,fp8,31,0.0323840007185936
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,63,0.03163733333349228
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,fp8,63,0.03264000018437704
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,127,0.03718400001525879
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,fp8,127,0.037402667105197906
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,255,0.05212266743183136
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,fp8,255,0.046240001916885376
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,1,0.04347200194994608
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,fp8,1,0.047226667404174805
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,3,0.04357333481311798
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,fp8,3,0.047024001677831016
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,7,0.043434664607048035
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,fp8,7,0.04682666560014089
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,15,0.04401599864164988
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,31,0.043968002001444496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,fp8,31,0.047269334395726524
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,63,0.049365331729253135
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,fp8,15,0.047024001677831016
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,fp8,63,0.04749333361784617
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,127,0.054101333022117615
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,fp8,127,0.06055466830730438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,255,0.0776693324247996
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,1,0.0685280015071233
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,fp8,1,0.07187200089295705
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,fp8,255,0.07046400010585785
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,3,0.06832533578077953
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,fp8,3,0.07225599884986877
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,fp8,7,0.07179733117421468
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,7,0.06806933383146922
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,15,0.06916266679763794
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,fp8,15,0.07268799841403961
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,31,0.07353599866231282
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,fp8,31,0.07177599767843883
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,63,0.07524799803892772
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,fp8,63,0.07613866527875264
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,127,0.0823359986146291
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,fp8,127,0.09445333480834961
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,255,0.12195733189582825
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,fp8,255,0.11013866464296977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,1,0.047797332207361855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,fp8,1,0.04915733138720194
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,fp8,3,0.04970666766166687
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,3,0.04798933366934458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,7,0.047882666190465294
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,fp8,7,0.049327999353408813
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,31,0.04781333108743032
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,fp8,31,0.04957333207130432
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,15,0.04809066653251648
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,fp8,15,0.049925332268079124
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,63,0.04993066688378652
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,fp8,63,0.05002133548259735
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,127,0.05751466751098633
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,1,0.059279998143514
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,fp8,127,0.05604266623655955
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,fp8,1,0.061941335598627724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,3,0.059450666109720864
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,fp8,3,0.06282133360703786
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,7,0.05961066484451294
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,fp8,7,0.06257066627343495
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,15,0.060266668597857155
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,fp8,15,0.062037333846092224
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,31,0.061535999178886414
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,fp8,31,0.06202666461467743
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,63,0.06704533100128174
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,fp8,63,0.06397866706053416
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,127,0.07307200133800507
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,1,0.08585600058237712
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,fp8,127,0.07709333300590515
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,fp8,1,0.09126399954160054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,3,0.08543999989827473
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,fp8,3,0.09074133634567261
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,7,0.08627733588218689
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,fp8,7,0.09188266595204671
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,15,0.08867733677228291
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,fp8,15,0.09231999516487122
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,fp8,31,0.09231467048327129
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,31,0.09275199969609578
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,63,0.09334400296211243
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,fp8,63,0.09801066915194194
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,127,0.10268800457318623
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,1,0.13515200217564902
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,fp8,127,0.11514133214950562
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,fp8,1,0.14050666491190592
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,3,0.13455466429392496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,fp8,3,0.14069333672523499
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,7,0.13659200072288513
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,15,0.14109866817792258
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,fp8,7,0.14191466569900513
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,fp8,15,0.14308266838391623
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,31,0.14268267154693604
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,1,0.011600000162919363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,3,0.011472000430027643
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,fp8,31,0.14642666776974997
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,1,0.012293333808581034
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,63,0.14550399780273438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,3,0.012410666793584824
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,7,0.012383999923865
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,fp8,63,0.1456160048643748
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,15,0.01259200026591619
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,7,0.011584000041087469
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,127,0.15869333346684775
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,31,0.011429333438475927
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,15,0.011440000186363855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,63,0.011578666667143503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,fp8,127,0.18160533905029297
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,31,0.012416000167528788
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,63,0.012784000486135483
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,127,0.01145600030819575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,255,0.012597333639860153
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,255,0.015066667149464289
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,511,0.014762666076421738
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,127,0.013573333621025085
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,1023,0.01646399994691213
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,1023,0.01852799952030182
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,511,0.016458666572968166
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,2047,0.021002667645613354
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,2047,0.02317333221435547
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,4095,0.029461334149042766
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,4095,0.033146666983763375
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,8191,0.04949333270390829
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,8191,0.05332266787687937
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,16383,0.08453866839408875
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,16383,0.09693333506584167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,1,0.011760000139474869
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,1,0.012576000144084295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,3,0.011594666788975397
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,3,0.012810666114091873
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,7,0.011706666400035223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,7,0.01249066616098086
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,15,0.011871999750534693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,15,0.012853333105643591
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,31,0.011936000237862269
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,31,0.012725333372751871
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,63,0.011653333902359009
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,63,0.012746666868527731
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,127,0.011770666887362799
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,127,0.0136266661187013
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,255,0.012885333349307379
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,255,0.01642666632930438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,511,0.015392000476519266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,511,0.016688000410795212
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,1023,0.017157333592573803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,1023,0.018858666221300762
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,8191,0.051216001311937966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,2047,0.02120000123977661
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,2047,0.023717333873112995
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,4095,0.031658666829268135
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,4095,0.03376533339420954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,1,0.013280000537633896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,8191,0.05605866511662801
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,16383,0.08634133140246074
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,1,0.012341332932313284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,16383,0.10123200217882793
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,3,0.011978667229413986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,3,0.01328533391157786
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,7,0.01246400053302447
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,7,0.013658666362365087
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,63,0.011861333002646765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,15,0.012351999680201212
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,15,0.01320533330241839
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,31,0.01211200033624967
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,31,0.013381333400805792
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,63,0.013359999905029932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,127,0.012597333639860153
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,127,0.015119999647140503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,255,0.013744000345468521
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,255,0.016282666474580765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,511,0.01526933287580808
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,511,0.01658133293191592
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,1023,0.017386666188637417
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,1023,0.01937066639463107
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,2047,0.022543999056021374
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,2047,0.02409599969784419
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,16383,0.10980799794197083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,4095,0.04264533519744873
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,4095,0.0353973334034284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,8191,0.07164266705513
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,8191,0.058730666836102806
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,16383,0.10079466303189595
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,1,0.012682666381200155
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,1,0.013248000293970108
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,3,0.01246400053302447
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,3,0.013104000439246496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,7,0.01221866657336553
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,7,0.01320533330241839
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,15,0.012736000120639801
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,63,0.012639999389648438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,15,0.013728000223636627
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,31,0.01268799975514412
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,31,0.01341333364446958
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,63,0.013568000247081121
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,127,0.012954667210578918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,127,0.015119999647140503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,255,0.014335999886194864
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,255,0.01643199970324834
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,511,0.016000000139077503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,511,0.01766933376590411
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,1023,0.018565333137909572
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,1023,0.01937599976857503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,2047,0.037861332297325134
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,2047,0.025727999707063038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,4095,0.05835733314355215
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,4095,0.04276266694068909
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,8191,0.10228266318639119
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,16383,0.12075199683507283
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,8191,0.07371733089288075
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,16383,0.18992000818252563
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,1,0.041637333730856575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,fp8,1,0.03958933303753535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,3,0.04475200176239014
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,fp8,3,0.03997866561015447
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,7,0.0444106658299764
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,fp8,7,0.04030400017897288
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,15,0.04897599915663401
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,fp8,15,0.0403466671705246
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,31,0.048384000857671104
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,fp8,31,0.042303999265034996
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,63,0.05004799862702688
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,1,0.07067733506361644
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,fp8,63,0.04797333478927612
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,fp8,1,0.06436799963315327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,3,0.07194666564464569
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,fp8,3,0.06451733410358429
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,7,0.07418133318424225
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,fp8,7,0.06559466818968455
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,15,0.07582933207352956
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,fp8,15,0.06756799916426341
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,31,0.0762613316377004
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,fp8,31,0.07382399837176006
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,63,0.07659733295440674
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,fp8,63,0.07526933153470357
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,1,0.12548800309499106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,fp8,1,0.11570666233698527
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,3,0.12896533807118735
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,fp8,3,0.11609599987665813
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,7,0.13080533345540366
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,fp8,7,0.11845333377520244
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,15,0.13060800234476724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,fp8,15,0.12498666842778523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,31,0.13105066617329916
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,fp8,31,0.12742933630943298
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,63,0.1339893341064453
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,1,0.206496000289917
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,fp8,63,0.1290880044301351
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,fp8,1,0.2044586737950643
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,3,0.20715733369191489
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,7,0.21332800388336182
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,fp8,3,0.20525334278742471
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,fp8,7,0.2088693380355835
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,15,0.2201333244641622
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,fp8,15,0.21888534228006998
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,31,0.22299732764561972
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,fp8,31,0.2163146734237671
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,fp8,63,0.22717867294947305
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,63,0.22985066970189413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,float16,1,0.08059733112653096
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,fp8,1,0.07948799928029378
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,float16,3,0.08142933249473572
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,fp8,3,0.08216533561547597
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,float16,7,0.08252800007661183
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,fp8,7,0.080485333998998
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,float16,15,0.08160000046094258
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,fp8,15,0.0811359981695811
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,fp8,31,0.08183999856313069
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,float16,31,0.08152533570925395
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,float16,1,0.13525333007176718
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,fp8,1,0.1272693375746409
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,float16,3,0.13543466726938883
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,fp8,3,0.12758933504422507
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,float16,7,0.13475733002026877
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,fp8,7,0.12845333417256674
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,float16,15,0.13578133781750998
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,fp8,15,0.12995200355847678
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,float16,1,0.24507200717926025
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,fp8,1,0.23228265841801962
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,float16,31,0.1371893286705017
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,fp8,31,0.13186132907867432
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,float16,3,0.24476265907287598
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,fp8,3,0.2334559957186381
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,fp8,7,0.23371734221776327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,float16,7,0.2456480065981547
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,fp8,15,0.23704000314076742
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,float16,31,0.24966933329900107
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,float16,15,0.24682666858037314
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,fp8,31,0.2373973329861959
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,float16,1,0.4057706594467163
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,fp8,1,0.4055146773656209
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,float16,3,0.4062826633453369
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,1,0.011802667131026586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,fp8,1,0.012879999975363413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,3,0.011749333391586939
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,float16,7,0.41706132888793945
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,fp8,7,0.41203200817108154
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,fp8,3,0.01302933320403099
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,fp8,7,0.013013333082199097
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,7,0.011781333635250727
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,float16,15,0.4238239924112956
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,15,0.011882666498422623
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,fp8,3,0.4054559866587321
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,fp8,15,0.01302933320403099
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,31,0.011744000017642975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,fp8,31,0.012725333372751871
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,fp8,63,0.013237333546082178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,fp8,127,0.014015999933083853
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,fp8,15,0.4123306671778361
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,127,0.011658667276302973
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,255,0.012789333860079447
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,63,0.011855999628702799
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,fp8,255,0.01522133375207583
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,511,0.015002666662136713
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,fp8,511,0.017312000195185345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,1023,0.019365333020687103
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,2047,0.02805333336194356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,fp8,2047,0.03246400008598963
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,fp8,1023,0.021930667261282604
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,4095,0.046426668763160706
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,8191,0.08259200056393941
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,fp8,4095,0.0510453333457311
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,fp8,8191,0.09635733564694722
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,1,0.012485332787036896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,fp8,1,0.013013333082199097
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,3,0.012128000458081564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,fp8,3,0.013370666652917862
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,7,0.012191999703645706
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,fp8,7,0.01340266689658165
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,15,0.012421333541472753
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,fp8,15,0.013269333789745966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,31,0.0122079998254776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,fp8,31,0.013386666774749756
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,63,0.012026666353146235
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,fp8,63,0.013258667041858038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,127,0.012565333396196365
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,fp8,127,0.01458666721979777
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,255,0.01370666672786077
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,fp8,255,0.015930666277805965
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,fp8,31,0.41748801867167157
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,511,0.01581866666674614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,float16,31,0.4305066665013631
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,fp8,511,0.017978666971127193
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,1023,0.02046400060256322
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,fp8,1023,0.02272533377011617
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,2047,0.02979733298222224
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,fp8,2047,0.03329066683848699
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,4095,0.049029335379600525
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,8191,0.08318933347860973
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,7,0.012234666695197424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,1,0.012485332787036896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,3,0.012357333054145178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,fp8,1,0.013717333475748697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,fp8,3,0.013653332988421122
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,fp8,7,0.013744000345468521
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,15,0.012213333199421564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,fp8,15,0.014053333550691605
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,31,0.012421333541472753
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,fp8,4095,0.053210665782292686
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,fp8,31,0.013525333255529404
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,fp8,8191,0.09963732957839966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,63,0.01251199965675672
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,fp8,63,0.013866666704416275
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,127,0.012655999511480331
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,fp8,127,0.01532799998919169
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,255,0.01403733342885971
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,fp8,255,0.015322666615247726
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,511,0.015743999431530636
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,fp8,511,0.01865600049495697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,2047,0.037087999284267426
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,fp8,1023,0.022895999252796173
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,fp8,2047,0.036389333506425224
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,1023,0.02088533341884613
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,4095,0.06061333417892456
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,fp8,4095,0.05779733260472616
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,8191,0.12544000148773193
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,1,0.012608000387748083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,3,0.012752000242471695
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,fp8,8191,0.09943999846776326
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,7,0.012613333761692047
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,fp8,7,0.013525333255529404
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,fp8,1,0.013855999956528345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,fp8,3,0.01393066719174385
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,fp8,15,0.013728000223636627
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,15,0.012741333494583765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,fp8,31,0.013754667093356451
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,31,0.012778667112191519
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,fp8,63,0.01402666668097178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,127,0.01331199953953425
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,63,0.012752000242471695
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,fp8,127,0.014943999548753103
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,255,0.01469333345691363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,fp8,1023,0.025008000433444977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,1023,0.03625600039958954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,fp8,255,0.01605333387851715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,fp8,2047,0.04050666590531667
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,2047,0.058330665032068886
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,4095,0.10143466790517171
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,fp8,4095,0.07310933371384938
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,511,0.017103999853134155
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,fp8,511,0.01844266677896182
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,8191,0.1893226703008016
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,fp8,8191,0.11190932989120483
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,1,0.011760000139474869
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,fp8,1,0.013023999830087027
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,3,0.012154666086037954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,fp8,3,0.01341333364446958
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,7,0.01227733368674914
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,fp8,15,0.013007999708255133
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,fp8,7,0.013301332791646322
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,15,0.012282667060693106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,31,0.012069333344697952
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,fp8,31,0.01350933313369751
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,63,0.011968000481526056
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,fp8,63,0.01381333296497663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,fp8,127,0.014736000448465347
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,255,0.014453332871198654
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,127,0.012319999436537424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,511,0.01579733317097028
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,fp8,255,0.015184000134468079
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,fp8,511,0.01809599995613098
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,1023,0.020031999796628952
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,fp8,1023,0.022629333039124806
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,2047,0.029738667110602062
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,fp8,2047,0.03244800120592117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,4095,0.04872000217437744
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,fp8,4095,0.0553706685702006
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,fp8,3,0.013855999956528345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,1,0.012448000411192576
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,fp8,1,0.013850666582584381
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,3,0.012341332932313284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,7,0.012133333832025528
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,fp8,7,0.013477332890033722
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,15,0.012416000167528788
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,fp8,15,0.013546666751305262
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,31,0.012560000022252401
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,fp8,31,0.013983999689420065
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,63,0.012240000069141388
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,fp8,63,0.013807999591032663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,127,0.012863999853531519
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,fp8,127,0.015008000036080679
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,255,0.0143306665122509
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,fp8,255,0.016016000260909397
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,511,0.016042667130629223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,fp8,511,0.01868266612291336
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,1023,0.02124800036350886
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,fp8,1023,0.02325333406527837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,2047,0.03765333443880081
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,fp8,2047,0.03397866586844126
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,4095,0.06151466568311056
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,fp8,4095,0.05702400207519531
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,1,0.012714666624863943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,fp8,1,0.014042666802803675
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,3,0.01267733300725619
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,fp8,3,0.013786666095256805
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,7,0.012666666259368261
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,fp8,7,0.014170666535695394
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,15,0.012576000144084295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,63,0.012661332885424295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,fp8,15,0.014127999544143677
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,127,0.012954667210578918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,31,0.012725333372751871
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,fp8,31,0.014021333307027817
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,fp8,63,0.01394133393963178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,fp8,127,0.015504000087579092
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,255,0.014495999862750372
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,fp8,255,0.01617066686352094
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,511,0.017125333348910015
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,fp8,511,0.018602666755517323
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,1023,0.035599999129772186
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,4095,0.10142933328946431
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,fp8,1023,0.025146665672461193
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,2047,0.05759466687838236
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,fp8,2047,0.04186133543650309
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,fp8,4095,0.06625600159168243
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,1,0.01716800034046173
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,fp8,1,0.017877332866191864
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,fp8,3,0.017829333742459614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,3,0.01647466669480006
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,7,0.016714667280515034
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,fp8,7,0.01807466646035512
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,15,0.016693333784739178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,fp8,15,0.01807466646035512
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,31,0.017765333255132038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,fp8,31,0.017914666483799618
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,63,0.016501333564519882
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,fp8,63,0.018437333405017853
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,127,0.01700266698996226
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,fp8,127,0.020928000410397846
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,255,0.021349333226680756
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,fp8,255,0.022218666970729828
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,511,0.03789333254098892
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,fp8,511,0.029546665648619335
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,1023,0.059343998630841575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,fp8,1023,0.04539733131726583
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,2047,0.10225066542625427
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,fp8,2047,0.06962133447329204
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,1,0.012815999488035837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,4095,0.18982932964960733
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,1,0.011349332829316458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,3,0.011509332805871964
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,fp8,4095,0.11680533488591512
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,3,0.012826666235923767
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,7,0.011642667154471079
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,7,0.012741333494583765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,15,0.011594666788975397
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,63,0.012666666259368261
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,15,0.012272000312805176
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,31,0.011450666934251785
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,31,0.012586666891972223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,63,0.011413333316644033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,127,0.011503999431928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,127,0.014293332894643148
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,255,0.012853333105643591
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,255,0.013807999591032663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,511,0.012960000584522883
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,511,0.014181333283583323
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,1023,0.013274667163689932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,1023,0.01498666654030482
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,2047,0.01441066712141037
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,2047,0.017429333180189133
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,4095,0.015743999431530636
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,4095,0.017024000485738117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,8191,0.01811733345190684
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,8191,0.0198186660806338
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,1,0.012159999459981918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,16383,0.022917332748572033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,16383,0.02515733242034912
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,1,0.011616000284751257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,3,0.011488000551859537
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,3,0.012506666282812754
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,7,0.01146666705608368
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,7,0.012272000312805176
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,15,0.01145600030819575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,15,0.012863999853531519
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,31,0.011530666301647821
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,255,0.012671999633312225
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,31,0.013045333325862885
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,63,0.011418666690587997
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,511,0.01292266696691513
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,63,0.012554666648308435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,1023,0.013445333888133367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,1023,0.015061333775520325
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,127,0.01157333329319954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,127,0.013621332744757334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,255,0.013967999567588171
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,511,0.014458666245142618
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,2047,0.014752000570297241
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,2047,0.01643199970324834
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,4095,0.016597333053747814
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,4095,0.01809599995613098
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,1,0.012666666259368261
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,8191,0.019050666441520054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,8191,0.021375998854637146
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,16383,0.023738667368888855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,16383,0.02605333427588145
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,1,0.011877333124478659
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,3,0.011690666278203329
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,3,0.01257066677014033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,7,0.011626667032639185
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,7,0.012752000242471695
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,15,0.011637333780527115
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,15,0.012815999488035837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,31,0.011962667107582092
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,31,0.012917333592971167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,63,0.011674666156371435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,63,0.012608000387748083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,127,0.011637333780527115
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,127,0.01368533323208491
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,255,0.012682666381200155
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,255,0.014117332796255747
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,511,0.013290667285521826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,511,0.014890667051076889
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,1023,0.014117332796255747
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,1023,0.01534933348496755
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,2047,0.015141333142916361
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,2047,0.016719999412695568
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,4095,0.016634666671355564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,4095,0.01794133335351944
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,8191,0.01894933357834816
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,8191,0.020703999946514767
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,16383,0.027221334477265675
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,16383,0.026394667724768322
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,1,0.01157333329319954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,1,0.012693333129088083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,3,0.01156266654531161
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,3,0.013077333569526672
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,7,0.011839999506870905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,7,0.012768000364303589
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,15,0.011861333002646765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,15,0.01251199965675672
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,31,0.011909333368142446
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,31,0.012517333030700684
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,63,0.011711999773979187
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,63,0.012655999511480331
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,127,0.01184533288081487
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,127,0.013898666948080063
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,255,0.012928000340859095
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,255,0.014938666174809137
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,511,0.013701333353916803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,511,0.015402667224407196
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,1023,0.014314666390419006
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,1023,0.015184000134468079
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,2047,0.015664000064134598
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,2047,0.01669866715868314
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,8191,0.020954666038354237
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,4095,0.01701333373785019
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,16383,0.02733866622050603
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,4095,0.017946666727463405
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,8191,0.019760000209013622
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,16383,0.0377813329299291
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,1,0.011343999455372492
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,1,0.012752000242471695
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,15,0.011493333925803503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,15,0.01292266696691513
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,7,0.011503999431928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,3,0.011567999919255575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,3,0.012975999464591345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,7,0.012202666451533636
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,31,0.011312000453472137
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,31,0.012810666114091873
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,63,0.011429333438475927
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,63,0.012901333471139273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,127,0.011509332805871964
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,127,0.01394133393963178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,255,0.012901333471139273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,255,0.013728000223636627
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,511,0.013130666067202887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,511,0.014522666732470194
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,1023,0.014442666123310724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,1023,0.015344000111023584
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,2047,0.016143999993801117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,2047,0.016373333831628162
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,4095,0.018522666146357853
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,4095,0.019205333044131596
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,8191,0.021322667598724365
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,8191,0.023813332120577495
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,16383,0.029893333713213604
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,16383,0.03405333310365677
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,1,0.011839999506870905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,1,0.012842666357755661
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,3,0.01163200040658315
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,3,0.012458667159080505
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,7,0.011600000162919363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,7,0.012762666990359625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,15,0.011770666887362799
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,15,0.012906666845083237
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,31,0.011760000139474869
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,31,0.012805332740147909
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,63,0.011695999652147293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,63,0.012538666526476542
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,127,0.011637333780527115
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,127,0.01394133393963178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,255,0.012858666479587555
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,255,0.014314666390419006
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,511,0.013477332890033722
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,511,0.015125333021084467
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,1023,0.014794666320085526
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,1023,0.016085332880417507
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,2047,0.016362667083740234
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,2047,0.016970666746298473
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,4095,0.01870399961868922
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,4095,0.019914666811625164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,8191,0.021967999637126923
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,1,0.012890666723251343
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,8191,0.024512000381946564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,16383,0.032058666149775185
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,16383,0.03438399980465571
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,1,0.011685332904259363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,3,0.011850666254758835
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,3,0.012810666114091873
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,7,0.011823999385039011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,7,0.012778667112191519
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,15,0.011861333002646765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,15,0.012896000097195307
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,31,0.011685332904259363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,31,0.012736000120639801
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,63,0.011941333611806234
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,63,0.012831999609867731
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,127,0.011823999385039011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,127,0.014197333405415217
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,255,0.01322666679819425
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,255,0.015029333531856537
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,511,0.01369599997997284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,511,0.014912000546852747
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,1023,0.015930666277805965
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,1023,0.016208000481128693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,2047,0.016666666915019352
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,2047,0.01757866640885671
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,4095,0.018735999862353008
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,4095,0.01998399943113327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,8191,0.025792000194390614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,1,0.012042666474978128
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,8191,0.02478933334350586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,16383,0.042821332812309265
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,16383,0.036101333796978
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,1,0.013141332815090815
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,7,0.01332266628742218
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,3,0.011930666863918304
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,3,0.013082666943470636
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,7,0.011978667229413986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,15,0.01210133358836174
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,63,0.012037333101034164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,15,0.013151999562978745
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,127,0.012250666817029318
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,31,0.012005332857370377
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,31,0.013274667163689932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,63,0.01314666618903478
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,127,0.014746667196353277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,511,0.015493333339691162
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,255,0.013541333377361298
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,255,0.014762666076421738
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,511,0.0141546664138635
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,1023,0.01600533351302147
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,1023,0.016751999656359356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,2047,0.016565332810084026
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,2047,0.017488000293572743
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,4095,0.019002666076024372
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,4095,0.020362666497627895
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,8191,0.037658666570981346
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,8191,0.027141332626342773
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,16383,0.05884799857934316
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,16383,0.04678933322429657
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,1,0.013386666774749756
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,fp8,1,0.013728000223636627
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,3,0.013253333667914072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,fp8,3,0.014170666535695394
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,fp8,7,0.013872000078360239
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,7,0.013056000073750814
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,15,0.012885333349307379
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,fp8,15,0.014149333039919535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,31,0.01314666618903478
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,fp8,31,0.014245333770910898
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,63,0.013354666531085968
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,fp8,63,0.014159999787807465
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,fp8,255,0.016085332880417507
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,511,0.016415999581416447
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,127,0.013397333522637686
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,fp8,127,0.015520000209410986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,255,0.01479999969402949
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,fp8,511,0.018805333723624546
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,1023,0.022111999491850536
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,fp8,1023,0.023472001155217487
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,fp8,1,0.01479999969402949
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,2047,0.0379573330283165
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,fp8,2047,0.034714666505654655
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,1,0.013295999417702356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,3,0.013343999783198038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,fp8,3,0.014592000593741735
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,7,0.013450667262077332
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,fp8,7,0.014725333700577417
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,15,0.013525333255529404
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,63,0.01358933374285698
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,fp8,15,0.014773332824309668
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,31,0.013327999661366144
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,fp8,31,0.01462399959564209
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,fp8,63,0.014959999670584997
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,127,0.014058666924635569
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,fp8,127,0.016074666132529575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,255,0.015717333803574245
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,fp8,255,0.016544000556071598
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,511,0.018640000373125076
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,fp8,511,0.019082666685183842
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,1023,0.03697066754102707
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,fp8,1023,0.02644266684850057
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,2047,0.058874666690826416
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,1,0.017375999440749485
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,fp8,2047,0.04257600009441376
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,fp8,1,0.018405333161354065
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,3,0.01741333305835724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,fp8,3,0.018160000443458557
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,7,0.017093333105246227
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,fp8,7,0.018511999398469925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,fp8,15,0.018522666146357853
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,15,0.017338667064905167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,fp8,31,0.01834133391578992
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,31,0.01729600007335345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,63,0.01762666677435239
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,fp8,63,0.018239999810854595
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,127,0.0173333336909612
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,fp8,127,0.021269333859284718
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,255,0.021359999974568684
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,fp8,255,0.0227360005180041
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,1023,0.06154666841030121
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,511,0.038704000413417816
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,fp8,511,0.029461334149042766
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,fp8,1023,0.04577599962552389
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,2047,0.10393066207567851
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,fp8,2047,0.07102933526039124
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,1,0.024165332317352295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,fp8,1,0.025813333690166473
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,3,0.023978665471076965
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,fp8,3,0.02569599946339925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,fp8,7,0.025727999707063038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,15,0.024266667664051056
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,7,0.023786666492621105
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,fp8,31,0.025637333591779072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,fp8,15,0.026015999416510265
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,31,0.024458666642506916
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,63,0.02443733314673106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,fp8,63,0.025888000925381977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,127,0.025616000096003216
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,fp8,127,0.029994666576385498
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,255,0.04223999877770742
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,fp8,255,0.03716266651948293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,1023,0.1053653359413147
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,fp8,511,0.05221866567929586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,511,0.06665066878000896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,fp8,1023,0.07566399872303009
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,2047,0.19006933768590292
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,1,0.011061333119869232
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,1,0.012400000045696894
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,fp8,2047,0.12341866890589397
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,3,0.01098666712641716
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,7,0.011525332927703857
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,3,0.01228800043463707
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,7,0.01191466674208641
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,15,0.010901333143313726
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,15,0.01232533281048139
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,31,0.010981333752473196
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,31,0.012538666526476542
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,63,0.012469333906968435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,63,0.011407999942700068
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,127,0.011168000598748526
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,127,0.013557333499193192
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,255,0.012485332787036896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,255,0.01402666668097178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,511,0.013744000345468521
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,511,0.015141333142916361
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,1023,0.015130666395028433
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,1023,0.01623999948302905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,2047,0.01635733370979627
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,2047,0.018981333822011948
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,4095,0.02073066681623459
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,4095,0.02310933421055476
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,8191,0.02937600016593933
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,8191,0.03347733368476232
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,16383,0.04808000226815542
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,16383,0.05283733208974203
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,1,0.01128000020980835
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,1,0.01232533281048139
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,3,0.011194666226704916
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,3,0.012383999923865
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,7,0.011317333827416102
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,7,0.012373333175977072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,15,0.011365332951148352
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,15,0.012858666479587555
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,31,0.011237333218256632
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,31,0.01250133290886879
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,63,0.011141333729028702
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,63,0.012400000045696894
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,127,0.011343999455372492
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,127,0.013712000101804733
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,1023,0.01704000060757001
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,255,0.012917333592971167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,255,0.015072000523408255
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,511,0.013797332843144735
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,511,0.015962666521469753
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,1023,0.015594666202863058
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,2047,0.01695466662446658
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,2047,0.019082666685183842
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,4095,0.021551998953024547
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,4095,0.02420266717672348
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,8191,0.031712000568707786
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,8191,0.03346666693687439
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,16383,0.050741334756215416
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,16383,0.05551466842492422
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,1,0.011695999652147293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,1,0.012469333906968435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,3,0.011722666521867117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,3,0.012624000509579977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,7,0.011706666400035223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,7,0.012389333297808966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,15,0.01179733375708262
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,15,0.012586666891972223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,31,0.011802667131026586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,31,0.012565333396196365
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,63,0.011760000139474869
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,63,0.012821332861979803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,127,0.011813333878914515
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,127,0.014256000518798828
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,255,0.013503999759753546
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,255,0.014746667196353277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,511,0.014901333798964819
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,511,0.016469333320856094
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,1023,0.016000000139077503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,1023,0.017231999586025875
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,2047,0.017157333592573803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,2047,0.019493332753578823
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,4095,0.02235200007756551
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,4095,0.024245334168275196
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,8191,0.03873066604137421
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,8191,0.03570666660865148
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,16383,0.060517330964406334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,16383,0.05861333509286245
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,1,0.012037333101034164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,1,0.013253333667914072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,3,0.012181332955757776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,3,0.013653332988421122
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,7,0.012416000167528788
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,7,0.013898666948080063
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,15,0.011946666985750198
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,15,0.013258667041858038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,31,0.012026666353146235
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,31,0.013061333447694778
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,63,0.012266666938861212
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,63,0.013194666554530462
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,127,0.012319999436537424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,127,0.014607999473810196
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,255,0.01351999988158544
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,255,0.014767999450365702
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,511,0.014762666076421738
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,511,0.016501333564519882
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,1023,0.01603200038274129
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,1023,0.017173333714405697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,2047,0.018464000274737675
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,2047,0.01951466624935468
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,4095,0.037130666275819145
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,4095,0.02513599892457326
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,8191,0.05865600208441416
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,8191,0.04678933322429657
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,16383,0.10213866829872131
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,16383,0.07557866473992665
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,1,0.01551466683546702
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,fp8,1,0.01632533346613248
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,3,0.015194666882356008
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,fp8,3,0.016528000434239704
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,7,0.015285332997639975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,fp8,7,0.01651200031240781
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,15,0.015392000476519266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,fp8,15,0.016490666816631954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,31,0.015439999600251516
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,fp8,31,0.01637866720557213
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,63,0.015552000453074774
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,fp8,63,0.016602666427691776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,127,0.016255999604860943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,fp8,127,0.017973333597183228
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,255,0.017765333255132038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,511,0.02093333254257838
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,fp8,255,0.018426666657129925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,fp8,511,0.021327999730904896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,1023,0.038736000657081604
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,1,0.018981333822011948
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,fp8,1023,0.029866665601730347
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,3,0.018906666586796444
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,fp8,3,0.020394666741291683
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,fp8,1,0.020154666155576706
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,7,0.01877333347996076
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,fp8,7,0.02080533280968666
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,15,0.018933333456516266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,fp8,15,0.020282667130231857
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,fp8,31,0.02060266708334287
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,fp8,63,0.020597333709398907
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,31,0.018960000326236088
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,127,0.01924266666173935
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,63,0.01934933289885521
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,fp8,127,0.022986667851607006
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,255,0.023610666394233704
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,fp8,255,0.02476799984773
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,511,0.04191466669241587
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,fp8,511,0.033530667424201965
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,1023,0.06313066681226094
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,1,0.025568000972270966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,fp8,1023,0.04966933528582255
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,fp8,1,0.02719466636578242
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,3,0.025413334369659424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,fp8,3,0.027461332579453785
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,7,0.025546667476495106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,fp8,7,0.02779199928045273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,fp8,15,0.02757333219051361
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,15,0.025722667574882507
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,31,0.025589334468046825
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,fp8,31,0.0271573339899381
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,63,0.025733334322770435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,fp8,63,0.027589333554108936
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,127,0.027866666515668232
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,fp8,127,0.03250666707754135
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,255,0.04615999758243561
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,fp8,255,0.03694933404525121
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,511,0.06682666639486949
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,fp8,511,0.05490666627883911
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,1,0.03841066608826319
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,fp8,1,0.04177066683769226
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,1023,0.10863999525705974
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,3,0.03846399982770284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,fp8,1023,0.0783733328183492
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,fp8,3,0.04110399881998698
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,fp8,7,0.04192533095677694
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,7,0.03812266637881597
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,15,0.0388373335202535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,31,0.039103999733924866
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,63,0.04020266731580099
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,fp8,63,0.04219200213750204
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,fp8,15,0.04146666576464971
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,127,0.046896000703175865
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,fp8,31,0.041536000867684685
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,fp8,127,0.05373333394527435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,255,0.07030400137106578
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,fp8,1023,0.1355839967727661
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,1023,0.19639466206232706
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,fp8,255,0.06698133548100789
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,511,0.11249066392580669
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,fp8,511,0.08886933326721191
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,1,0.022986667851607006
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,fp8,1,0.024906667570273083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,3,0.022522665560245514
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,fp8,3,0.024826665719350178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,7,0.02266666789849599
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,fp8,7,0.02495466669400533
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,fp8,15,0.025263999899228413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,fp8,31,0.024933333198229473
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,31,0.022890667120615642
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,63,0.022858666876951855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,15,0.0229120006163915
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,fp8,63,0.024890666206677754
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,127,0.023904000719388325
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,255,0.030133334298928578
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,fp8,127,0.02717866748571396
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,fp8,255,0.029002666473388672
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,1,0.028949332733949024
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,fp8,1,0.03148266673088074
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,3,0.028890666862328846
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,fp8,3,0.03123733401298523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,7,0.028912000358104706
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,fp8,7,0.03163733333349228
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,15,0.029103999336560566
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,fp8,15,0.03175999969244003
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,31,0.029258665939172108
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,fp8,31,0.031301334500312805
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,63,0.029290666182835896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,fp8,63,0.03145600110292435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,127,0.034789333740870156
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,fp8,127,0.0360000009338061
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,255,0.049679999550183616
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,1,0.041984001795450844
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,fp8,255,0.04345066845417023
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,fp8,1,0.0447626660267512
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,3,0.041696002086003624
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,fp8,3,0.04480533301830292
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,7,0.042122667034467064
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,fp8,7,0.045498669147491455
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,15,0.04195199906826019
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,fp8,15,0.04529599845409393
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,31,0.042223999897638954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,fp8,31,0.04491200049718221
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,63,0.045696000258127846
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,fp8,63,0.0451200008392334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,127,0.05367999772230784
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,fp8,127,0.058634668588638306
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,255,0.07529599964618683
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,1,0.06753600140412648
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,fp8,255,0.06960000097751617
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,fp8,1,0.07141333321730296
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,3,0.06729066868623097
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,fp8,3,0.07194666564464569
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,7,0.06780800223350525
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,fp8,7,0.07259200016657512
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,fp8,15,0.07217599948247273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,fp8,31,0.07180266578992207
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,15,0.06890666484832764
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,63,0.0745066652695338
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,fp8,63,0.07522666454315186
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,31,0.07230933507283528
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,127,0.08238933483759563
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,fp8,127,0.09507200121879578
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,255,0.12202133735020955
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,fp8,255,0.1099679966767629
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,1,0.04376000165939331
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,fp8,1,0.044768000642458596
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,3,0.043231998880704246
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,fp8,3,0.04524266719818115
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,7,0.043621331453323364
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,fp8,7,0.0452159990866979
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,31,0.04380266865094503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,fp8,31,0.045125335454940796
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,15,0.043621331453323364
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,63,0.0443200021982193
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,fp8,15,0.04470933477083842
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,fp8,63,0.04534400006135305
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,127,0.05231466889381409
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,fp8,127,0.049770668148994446
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,1,0.055104002356529236
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,fp8,1,0.05738666653633118
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,3,0.05527466535568237
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,fp8,3,0.05721066892147064
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,7,0.05499200026194254
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,fp8,7,0.058448001742362976
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,15,0.05604266623655955
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,fp8,15,0.05807999769846598
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,31,0.05620799958705902
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,fp8,31,0.05797866483529409
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,63,0.06277866661548615
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,fp8,63,0.05754133562246958
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,127,0.0687360018491745
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,fp8,127,0.0719893326361974
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,1,0.0814879983663559
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,fp8,1,0.08628267049789429
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,3,0.08131733536720276
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,fp8,3,0.08638933300971985
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,7,0.08117866516113281
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,fp8,15,0.08683199683825175
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,fp8,7,0.08751466870307922
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,31,0.0890773336092631
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,15,0.08302933474381764
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,fp8,31,0.08724266290664673
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,63,0.09001599748929341
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,fp8,63,0.09330133597056071
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,127,0.09934399525324504
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,1,0.1334986686706543
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,fp8,127,0.11030399799346924
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,fp8,1,0.13988266388575235
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,3,0.13429866234461466
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,fp8,3,0.139984001715978
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,7,0.1332373321056366
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,fp8,7,0.1397546629110972
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,fp8,15,0.14225600163141885
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,15,0.14153599739074707
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,fp8,31,0.14641599853833517
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,31,0.14289066195487976
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,1,0.011663999408483505
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,fp8,63,0.1478613317012787
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,63,0.1448906660079956
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,1,0.012543999900420507
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,3,0.011285333583752314
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,7,0.011621333658695221
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,3,0.012458667159080505
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,15,0.012495999534924826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,31,0.011391999820868174
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,7,0.012618667135636011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,31,0.013034666577974955
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,127,0.011519999553759893
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,63,0.011557333171367645
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,15,0.011370666325092316
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,63,0.012602667013804117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,127,0.013722666849692663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,127,0.15573867162068686
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,fp8,127,0.17933332920074463
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,255,0.012602667013804117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,255,0.01482133318980535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,511,0.01463466634353002
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,511,0.01600533351302147
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,1023,0.016554666062196095
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,1023,0.01858666663368543
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,4095,0.029482667644818623
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,4095,0.03317866722742716
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,8191,0.04785066843032837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,8191,0.052655999859174095
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,16383,0.08418132861455281
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,16383,0.09556266665458679
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,2047,0.02090666691462199
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,1,0.011706666400035223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,2047,0.023157333334287006
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,1,0.012624000509579977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,3,0.011706666400035223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,3,0.012896000097195307
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,7,0.011813333878914515
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,7,0.012917333592971167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,15,0.01173866664369901
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,15,0.012863999853531519
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,31,0.011557333171367645
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,31,0.012624000509579977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,63,0.011706666400035223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,63,0.012869333227475485
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,127,0.011882666498422623
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,127,0.013951999445756277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,255,0.013007999708255133
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,255,0.01640533283352852
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,511,0.014639999717473984
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,511,0.01648533344268799
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,1023,0.018298666924238205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,1023,0.018960000326236088
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,2047,0.021317332983016968
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,2047,0.02385066697994868
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,4095,0.029978667696317036
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,4095,0.03348266581694285
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,8191,0.04956266780694326
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,8191,0.054602667689323425
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,16383,0.08795199791590373
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,16383,0.10177600383758545
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,1,0.01221866657336553
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,1,0.01332266628742218
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,15,0.012298667182525
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,3,0.011909333368142446
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,3,0.013776000589132309
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,7,0.011829332758982977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,7,0.013477332890033722
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,15,0.013248000293970108
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,31,0.012106666962305704
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,31,0.01333333303531011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,63,0.01198400060335795
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,63,0.013605333864688873
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,127,0.012042666474978128
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,127,0.014933332800865173
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,255,0.01394133393963178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,255,0.016421332955360413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,511,0.015194666882356008
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,511,0.016458666572968166
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,1023,0.017360000560681026
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,1023,0.019194666296243668
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,4095,0.03645866612593333
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,2047,0.025040000677108765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,8191,0.058143998185793556
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,2047,0.023775999744733173
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,4095,0.03916800022125244
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,8191,0.062047998110453285
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,16383,0.10922666390736897
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,16383,0.10135466853777568
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,1,0.012453333785136541
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,1,0.013861333330472311
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,3,0.012373333175977072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,3,0.01349866638580958
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,7,0.011898666620254517
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,7,0.013317332913478216
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,15,0.012191999703645706
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,15,0.013594667116800943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,31,0.012453333785136541
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,31,0.01351999988158544
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,63,0.012117333710193634
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,255,0.013807999591032663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,63,0.01322666679819425
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,127,0.012565333396196365
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,127,0.014901333798964819
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,255,0.017583999782800674
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,511,0.015471999843915304
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,511,0.01798933371901512
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,1023,0.01841066653529803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,4095,0.058703998724619545
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,1023,0.01940800001223882
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,2047,0.037392000357309975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,2047,0.027962667246659596
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,4095,0.04289066791534424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,8191,0.10335999727249146
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,8191,0.06701866785685222
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,16383,0.18968000014623007
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,16383,0.1311946709950765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,1,0.03920000046491623
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,fp8,1,0.0391839991013209
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,3,0.039290666580200195
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,fp8,3,0.039877332746982574
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,fp8,7,0.03972800076007843
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,15,0.04201066493988037
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,7,0.03953066716591517
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,fp8,15,0.040005333721637726
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,31,0.04563199977080027
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,63,0.047968000173568726
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,fp8,31,0.03988266736268997
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,1,0.06538133323192596
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,fp8,63,0.04351999859015147
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,fp8,1,0.06362666686375935
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,fp8,3,0.06340266764163971
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,3,0.06629333396752675
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,fp8,7,0.06356800099213918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,7,0.06748799979686737
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,15,0.0728053351243337
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,fp8,15,0.06506133576234181
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,31,0.07453866799672444
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,fp8,31,0.06888533135255177
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,63,0.07495999832948048
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,fp8,63,0.07415999968846639
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,1,0.11774399876594543
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,fp8,1,0.11328533291816711
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,fp8,3,0.11400000254313152
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,3,0.12089600165685017
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,7,0.128629336754481
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,fp8,7,0.11462933818499248
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,15,0.1300266683101654
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,fp8,15,0.11964799960454305
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,31,0.13081600268681845
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,fp8,31,0.1267680029074351
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,1,0.2216906746228536
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,63,0.1318933367729187
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,fp8,1,0.2063093384106954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,fp8,63,0.12839999794960022
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,3,0.22963200012842813
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,fp8,3,0.20800000429153442
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,7,0.23270932833353677
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,fp8,7,0.21563732624053955
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,15,0.2341759999593099
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,fp8,15,0.22402133544286093
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,31,0.23301867643992105
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,fp8,31,0.22355733315149942
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,63,0.23663999636967978
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,fp8,63,0.2265119949976603
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,float16,1,0.07973333199818929
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,fp8,1,0.07413866619269054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,float16,3,0.07819733520348866
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,fp8,3,0.07517333328723907
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,float16,7,0.07786133388678233
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,fp8,7,0.07753066718578339
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,float16,15,0.07937066753705342
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,fp8,15,0.07824533184369405
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,float16,1,0.13172800342241922
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,fp8,31,0.07985599835713704
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,float16,31,0.07870399951934814
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,fp8,1,0.12500799695650736
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,fp8,3,0.1258133351802826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,float16,3,0.13168000181516012
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,float16,7,0.13269866506258646
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,fp8,7,0.12666133046150208
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,float16,15,0.1332533359527588
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,fp8,15,0.127920001745224
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,float16,31,0.1335093379020691
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,float16,1,0.24333866437276205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,fp8,1,0.23067200183868408
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,fp8,31,0.12987200419108072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,float16,3,0.24332799514134726
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,fp8,3,0.23306665817896524
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,float16,7,0.24355733394622803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,fp8,7,0.23456533749898276
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,float16,15,0.24393065770467123
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,fp8,15,0.2350613276163737
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,fp8,31,0.23719465732574463
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,float16,31,0.24463466803232828
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,float16,1,0.447925329208374
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,fp8,1,0.4282933473587036
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,float16,3,0.44795199235280353
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,fp8,3,0.42925333976745605
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,1,0.011850666254758835
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,fp8,1,0.01302933320403099
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,3,0.011765333513418833
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,float16,7,0.4482186635335286
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,fp8,3,0.012853333105643591
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,fp8,7,0.42853331565856934
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,7,0.011781333635250727
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,fp8,7,0.012757333616415659
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,15,0.011802667131026586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,float16,15,0.4500053326288859
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,fp8,15,0.012618667135636011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,fp8,31,0.012869333227475485
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,31,0.01179733375708262
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,63,0.011829332758982977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,fp8,15,0.4296799898147583
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,fp8,127,0.014053333550691605
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,fp8,63,0.012741333494583765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,127,0.011973333855470022
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,255,0.013050666699806849
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,fp8,255,0.014730667074521383
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,511,0.015146666516860327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,fp8,511,0.01727466657757759
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,fp8,1023,0.02231466770172119
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,1023,0.01960533360640208
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,2047,0.02824000020821889
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,fp8,4095,0.05101866523424784
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,8191,0.08276799817879994
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,fp8,2047,0.031888000667095184
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,4095,0.04677866895993551
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,fp8,8191,0.09179733196894328
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,1,0.012341332932313284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,fp8,1,0.013471999516089758
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,3,0.012191999703645706
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,fp8,3,0.013482666263977686
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,7,0.012293333808581034
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,fp8,7,0.013381333400805792
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,fp8,31,0.4331893523534139
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,15,0.012469333906968435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,fp8,15,0.013477332890033722
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,31,0.011823999385039011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,fp8,31,0.013376000026861826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,float16,31,0.4514186779658
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,63,0.012469333906968435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,fp8,63,0.013631999492645264
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,127,0.012330666184425354
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,255,0.013973332941532135
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,fp8,255,0.015354666858911514
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,511,0.015450666348139444
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,fp8,127,0.014896000425020853
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,fp8,511,0.01802666609485944
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,fp8,1023,0.022831998765468597
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,2047,0.029391999046007793
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,fp8,2047,0.03287466615438461
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,1023,0.019914666811625164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,4095,0.05124266445636749
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,fp8,4095,0.05499200026194254
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,8191,0.08498666683832805
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,fp8,8191,0.1006666620572408
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,1,0.012282667060693106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,fp8,7,0.013365333278973898
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,fp8,1,0.013978666315476099
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,3,0.012479999413092932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,fp8,3,0.013653332988421122
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,7,0.012357333054145178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,15,0.012469333906968435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,fp8,15,0.013898666948080063
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,31,0.012341332932313284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,fp8,31,0.01341333364446958
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,fp8,127,0.015274666249752045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,63,0.01257066677014033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,fp8,63,0.013850666582584381
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,127,0.012938667088747025
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,255,0.014554666976133982
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,fp8,255,0.015722667177518208
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,fp8,511,0.01852799952030182
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,511,0.015813333292802174
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,1023,0.02147199958562851
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,fp8,1023,0.023120000958442688
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,2047,0.03624533365170161
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,fp8,2047,0.03435733417669932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,4095,0.060645331939061485
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,fp8,4095,0.05743999779224396
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,8191,0.10460799932479858
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,fp8,8191,0.09973866740862529
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,1,0.012586666891972223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,fp8,1,0.01381333296497663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,3,0.01267733300725619
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,fp8,3,0.013514666507641474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,7,0.012432000289360682
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,fp8,7,0.013658666362365087
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,15,0.012634667257467905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,fp8,15,0.01404800017674764
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,31,0.012586666891972223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,fp8,31,0.013946666071812311
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,63,0.012448000411192576
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,fp8,63,0.013893333574136099
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,fp8,255,0.016048000504573185
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,127,0.012778667112191519
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,fp8,127,0.015504000087579092
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,255,0.015008000036080679
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,511,0.01721599946419398
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,fp8,511,0.018394666413466137
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,fp8,2047,0.041162667175134025
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,1023,0.03550933301448822
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,fp8,1023,0.02407466620206833
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,2047,0.05735466877619425
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,4095,0.10074667135874431
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,fp8,4095,0.0660693347454071
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,1,0.01231466606259346
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,8191,0.1888373295466105
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,fp8,8191,0.11319466431935628
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,fp8,1,0.013269333789745966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,3,0.011999999483426413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,fp8,3,0.013290667285521826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,7,0.0122079998254776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,fp8,7,0.01320533330241839
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,fp8,15,0.013130666067202887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,15,0.01180800050497055
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,31,0.0124746672809124
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,fp8,31,0.013248000293970108
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,63,0.012229333321253458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,fp8,63,0.013295999417702356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,127,0.012202666451533636
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,fp8,127,0.014730667074521383
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,255,0.013584000368913015
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,fp8,255,0.01505600040157636
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,511,0.01607999950647354
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,fp8,511,0.017898666361967724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,1023,0.021525333325068157
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,fp8,1023,0.02271466702222824
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,2047,0.029616000751654308
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,fp8,2047,0.03264000018437704
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,4095,0.04961599906285604
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,fp8,3,0.013839999834696451
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,fp8,4095,0.052576000491778054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,1,0.012602667013804117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,fp8,1,0.013466666142145792
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,3,0.012378666549921036
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,7,0.012479999413092932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,fp8,7,0.01340266689658165
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,fp8,15,0.013610667238632837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,15,0.01259200026591619
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,31,0.012309333930412928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,fp8,31,0.013957332819700241
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,63,0.013546666751305262
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,fp8,63,0.013440000514189402
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,127,0.012885333349307379
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,fp8,127,0.015247999380032221
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,255,0.01441066712141037
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,fp8,255,0.015967999895413715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,511,0.015909332782030106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,fp8,511,0.01871466636657715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,1023,0.0206986665725708
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,fp8,1023,0.0225600004196167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,2047,0.03669333209594091
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,fp8,2047,0.03473066786924998
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,4095,0.06009600063165029
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,fp8,4095,0.0573226660490036
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,1,0.012538666526476542
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,fp8,1,0.014117332796255747
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,3,0.012709333250919977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,fp8,3,0.01368533323208491
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,7,0.012709333250919977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,fp8,7,0.013936000565687815
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,15,0.012650666137536367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,fp8,15,0.013573333621025085
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,31,0.012576000144084295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,fp8,31,0.014111999422311783
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,63,0.012602667013804117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,fp8,63,0.014127999544143677
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,127,0.012970666090647379
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,fp8,127,0.015520000209410986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,255,0.01463466634353002
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,fp8,255,0.015813333292802174
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,511,0.01658133293191592
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,fp8,511,0.018746666610240936
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,2047,0.057333335280418396
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,1023,0.036133334040641785
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,fp8,1023,0.024165332317352295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,fp8,2047,0.04112533231576284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,4095,0.10076799988746643
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,fp8,4095,0.06594133377075195
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,1,0.01674666628241539
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,fp8,1,0.018378666291634243
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,3,0.016469333320856094
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,fp8,3,0.0179626668492953
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,7,0.01639466608564059
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,fp8,7,0.01783466711640358
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,15,0.01664000004529953
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,fp8,15,0.01814933369557063
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,31,0.016693333784739178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,fp8,31,0.018144000321626663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,63,0.016656000167131424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,fp8,63,0.017845333864291508
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,127,0.01692266638080279
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,fp8,255,0.02235200007756551
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,fp8,127,0.021040000021457672
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,255,0.021295999487241108
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,511,0.03855466594298681
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,fp8,511,0.028442665934562683
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,1023,0.05963733295599619
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,fp8,1023,0.044821331898371376
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,2047,0.10249599814414978
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,fp8,2047,0.0689279983441035
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,4095,0.1896160046259562
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,1,0.011503999431928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,1,0.01251199965675672
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,fp8,4095,0.11709333459536235
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,3,0.011413333316644033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,3,0.012666666259368261
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,7,0.01146666705608368
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,7,0.012522666404644648
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,63,0.012794667234023413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,15,0.011429333438475927
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,15,0.012400000045696894
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,31,0.01145600030819575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,31,0.012351999680201212
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,63,0.011365332951148352
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,127,0.011434666812419891
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,127,0.01402666668097178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,255,0.012752000242471695
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,2047,0.014671999961137772
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,255,0.014202666779359182
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,511,0.012730666746695837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,511,0.014192000031471252
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,1023,0.013002666334311167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,1023,0.014362666755914688
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,2047,0.01587733378012975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,4095,0.015770666301250458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,4095,0.01714666684468587
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,8191,0.017935999979575474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,8191,0.01961600035429001
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,16383,0.022917332748572033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,16383,0.024703999360402424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,1,0.011429333438475927
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,1,0.012624000509579977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,3,0.011530666301647821
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,3,0.013072000195582708
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,7,0.011429333438475927
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,7,0.012442667037248611
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,15,0.011424000064531961
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,15,0.012357333054145178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,31,0.011434666812419891
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,31,0.012725333372751871
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,63,0.01156266654531161
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,63,0.012741333494583765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,127,0.01137599969903628
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,127,0.013487999637921652
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,255,0.012608000387748083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,255,0.013562666873137156
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,511,0.012842666357755661
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,4095,0.016517333686351776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,511,0.014271999398867289
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,1023,0.013461332768201828
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,1023,0.015002666662136713
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,2047,0.014522666732470194
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,2047,0.01640533283352852
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,4095,0.017914666483799618
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,8191,0.019093333433071773
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,8191,0.020768000433842342
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,16383,0.023658665517965954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,16383,0.02611733227968216
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,1,0.01180800050497055
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,1,0.012725333372751871
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,3,0.01179733375708262
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,3,0.0124746672809124
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,7,0.011765333513418833
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,7,0.012762666990359625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,15,0.011786667009194693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,15,0.01333333303531011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,31,0.011744000017642975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,31,0.012608000387748083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,63,0.011530666301647821
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,63,0.012645332763592402
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,127,0.011711999773979187
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,127,0.013898666948080063
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,255,0.012837332983811697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,255,0.014042666802803675
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,511,0.01303999995191892
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,511,0.014767999450365702
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,1023,0.013909333695967993
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,1023,0.01570133368174235
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,2047,0.015578666081031164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,2047,0.016890666137139004
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,4095,0.016938666502634685
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,4095,0.017535999417304993
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,8191,0.018810667097568512
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,8191,0.02048533285657565
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,16383,0.02475200096766154
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,16383,0.026335999369621277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,1,0.011786667009194693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,1,0.012794667234023413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,3,0.011776000261306763
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,15,0.013114667187134424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,3,0.01231466606259346
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,7,0.011648000528415045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,7,0.012634667257467905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,15,0.011898666620254517
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,31,0.011727999895811081
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,31,0.012736000120639801
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,63,0.011877333124478659
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,63,0.012362666428089142
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,127,0.011674666156371435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,127,0.013962666193644205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,255,0.012991999586423239
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,255,0.014965333044528961
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,511,0.013872000078360239
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,511,0.015583999454975128
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,1023,0.014256000518798828
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,1023,0.01568000018596649
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,2047,0.01580799991885821
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,2047,0.01700266698996226
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,4095,0.016794666647911072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,8191,0.019882666567961376
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,4095,0.018181333939234417
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,8191,0.020725333442290623
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,3,0.011434666812419891
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,16383,0.03730666637420654
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,16383,0.027456000447273254
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,1,0.011530666301647821
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,1,0.012826666235923767
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,3,0.012437333663304647
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,7,0.011546666423479715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,7,0.012730666746695837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,15,0.011333333949247995
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,31,0.01145600030819575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,15,0.012266666938861212
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,31,0.013061333447694778
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,63,0.011493333925803503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,63,0.0124746672809124
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,127,0.011610666910807291
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,127,0.014346666634082794
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,255,0.012543999900420507
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,255,0.01381333296497663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,511,0.012906666845083237
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,511,0.014618666221698126
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,1023,0.01434133326013883
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,1023,0.015184000134468079
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,2047,0.015541333705186844
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,2047,0.01684800038735072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,4095,0.01748266691962878
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,4095,0.019199999670187633
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,8191,0.02102400114138921
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,8191,0.023669332265853882
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,16383,0.029877332349618275
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,16383,0.033370666205883026
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,1,0.011594666788975397
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,1,0.013290667285521826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,3,0.011498666057984034
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,3,0.01267733300725619
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,7,0.011477333803971609
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,7,0.012863999853531519
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,15,0.011760000139474869
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,15,0.01258133351802826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,31,0.011711999773979187
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,31,0.013503999759753546
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,63,0.01146666705608368
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,63,0.012586666891972223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,127,0.011488000551859537
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,127,0.013663999736309052
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,255,0.012981332838535309
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,255,0.013973332941532135
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,511,0.01332266628742218
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,511,0.015360000232855478
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,1023,0.014655999839305878
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,1023,0.0162773331006368
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,2047,0.016293333222468693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,2047,0.017231999586025875
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,4095,0.01826133330663045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,4095,0.01988799994190534
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,8191,0.022111999491850536
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,8191,0.024271999796231587
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,16383,0.03159466634194056
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,16383,0.03409600009520849
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,1,0.011717333147923151
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,1,0.012810666114091873
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,3,0.011781333635250727
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,3,0.013056000073750814
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,7,0.011861333002646765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,7,0.012655999511480331
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,15,0.011658667276302973
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,15,0.012768000364303589
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,31,0.01209066684047381
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,31,0.013013333082199097
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,127,0.013951999445756277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,63,0.011834666132926941
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,63,0.012768000364303589
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,127,0.011887999872366587
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,255,0.013183999806642532
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,255,0.014757333944241205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,511,0.013877333452304205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,511,0.01522133375207583
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,1023,0.015279999623696009
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,1023,0.016506666938463848
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,2047,0.01647466669480006
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,2047,0.01721599946419398
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,4095,0.0186666672428449
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,4095,0.019930666933457058
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,8191,0.02254933367172877
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,8191,0.024703999360402424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,16383,0.03798400113979975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,16383,0.03661333272854487
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,1,0.012042666474978128
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,1,0.012960000584522883
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,3,0.012165332833925882
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,31,0.011936000237862269
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,3,0.012986666212479273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,7,0.011946666985750198
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,63,0.01232533281048139
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,7,0.013370666652917862
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,15,0.011962667107582092
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,15,0.013013333082199097
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,31,0.013183999806642532
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,63,0.013050666699806849
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,127,0.01228800043463707
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,127,0.014501333236694336
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,255,0.013717333475748697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,255,0.015360000232855478
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,511,0.013957332819700241
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,511,0.01533866673707962
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,1023,0.015925332903862
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,1023,0.016656000167131424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,2047,0.01693333312869072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,2047,0.017386666188637417
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,4095,0.01972266659140587
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,4095,0.02041600023706754
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,8191,0.038005332152048744
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,8191,0.025205334027608235
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,16383,0.05968533456325531
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,3,0.01312000056107839
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,16383,0.04301866888999939
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,1,0.013023999830087027
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,7,0.01360000049074491
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,fp8,7,0.01404800017674764
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,fp8,1,0.013663999736309052
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,15,0.013381333400805792
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,fp8,3,0.014181333283583323
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,fp8,15,0.013818666338920593
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,31,0.012975999464591345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,fp8,31,0.013855999956528345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,63,0.013301332791646322
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,fp8,63,0.013994666437307993
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,511,0.01676799977819125
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,127,0.013125333935022354
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,fp8,511,0.018453333526849747
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,fp8,127,0.015279999623696009
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,255,0.015322666615247726
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,fp8,255,0.015978666643301647
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,1023,0.02534399926662445
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,fp8,1023,0.023365333676338196
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,2047,0.03760000069936117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,1,0.013461332768201828
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,fp8,2047,0.03454933315515518
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,fp8,1,0.014650666465361914
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,3,0.01349866638580958
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,fp8,3,0.014741333822409311
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,7,0.013306666165590286
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,31,0.013232000172138214
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,fp8,7,0.01440000037352244
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,15,0.013359999905029932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,fp8,15,0.014826666563749313
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,fp8,31,0.014767999450365702
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,63,0.013536000003417334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,fp8,63,0.014874666929244995
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,127,0.01381333296497663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,fp8,127,0.015594666202863058
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,511,0.0182239996890227
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,255,0.015098666151364645
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,fp8,255,0.016517333686351776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,1023,0.03709866603215536
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,fp8,511,0.019413333386182785
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,fp8,1023,0.025034666061401367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,2047,0.05795200169086456
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,1,0.01720533271630605
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,fp8,1,0.018288000176350277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,fp8,2047,0.042949333786964417
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,3,0.017173333714405697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,fp8,3,0.018624000251293182
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,7,0.01717866708834966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,15,0.017024000485738117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,fp8,7,0.018058666338523228
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,fp8,15,0.01836799954374631
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,fp8,31,0.018437333405017853
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,fp8,63,0.018432000031073887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,31,0.017386666188637417
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,63,0.017407999684413273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,127,0.017407999684413273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,fp8,127,0.021013334393501282
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,255,0.021338666478792827
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,fp8,255,0.022629333039124806
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,511,0.038912000755469
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,fp8,511,0.029882666965325672
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,1023,0.06058666606744131
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,2047,0.10423466563224792
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,fp8,1023,0.045647998650868736
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,fp8,2047,0.07089066505432129
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,1,0.02405333270629247
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,fp8,1,0.02584533393383026
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,3,0.02420799930890401
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,fp8,3,0.025973332424958546
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,15,0.024154665569464367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,7,0.023775999744733173
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,fp8,15,0.025701334079106648
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,fp8,7,0.02531733363866806
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,31,0.024346667031447094
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,fp8,31,0.02589866767326991
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,63,0.024336000283559162
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,fp8,63,0.025648000339667004
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,127,0.025311999022960663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,fp8,127,0.03035199890534083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,255,0.042437334855397545
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,fp8,255,0.03457599878311157
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,511,0.0635040005048116
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,1023,0.10563199718793233
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,fp8,511,0.05220800141493479
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,fp8,1023,0.07538666824499766
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,1,0.01090666651725769
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,fp8,2047,0.12344533205032349
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,1,0.01257066677014033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,2047,0.19050133228302002
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,3,0.011183999478816986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,3,0.01201066623131434
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,7,0.01138666644692421
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,7,0.011802667131026586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,15,0.011061333119869232
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,15,0.012400000045696894
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,31,0.01091733326514562
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,63,0.01099733387430509
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,31,0.012442667037248611
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,63,0.012058666596810022
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,127,0.011328000575304031
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,127,0.013232000172138214
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,255,0.012543999900420507
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,255,0.01393066719174385
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,511,0.013605333864688873
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,511,0.01515199989080429
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,1023,0.015029333531856537
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,1023,0.015962666521469753
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,2047,0.01646399994691213
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,2047,0.018794666975736618
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,4095,0.02046400060256322
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,4095,0.02314666658639908
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,1,0.011413333316644033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,8191,0.029125332832336426
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,8191,0.03319466610749563
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,16383,0.04831466575463613
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,16383,0.05222400029500326
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,1,0.012138667205969492
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,3,0.011231999844312668
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,3,0.012426666915416718
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,7,0.011349332829316458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,7,0.0122079998254776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,15,0.011253333340088526
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,15,0.01228800043463707
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,31,0.01126933346192042
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,31,0.012149333953857422
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,63,0.011215999722480774
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,63,0.012458667159080505
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,127,0.011359999577204386
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,127,0.01358933374285698
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,255,0.01292266696691513
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,255,0.01482133318980535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,511,0.013962666193644205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,511,0.01570133368174235
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,1023,0.015450666348139444
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,1023,0.016735999534527462
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,2047,0.01687466725707054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,2047,0.01913600042462349
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,4095,0.02146666745344798
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,16383,0.05611200133959452
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,4095,0.023552000522613525
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,8191,0.02945599953333537
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,8191,0.03352533280849457
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,16383,0.04967466493447622
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,1,0.011674666156371435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,1,0.012752000242471695
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,3,0.011685332904259363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,3,0.012416000167528788
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,7,0.011509332805871964
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,7,0.012448000411192576
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,15,0.011744000017642975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,15,0.012741333494583765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,31,0.011776000261306763
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,31,0.012741333494583765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,63,0.011605333536863327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,63,0.012586666891972223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,127,0.011514666179815928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,127,0.01441066712141037
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,255,0.013455999394257864
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,255,0.01463466634353002
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,511,0.015413332730531693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,511,0.01616000011563301
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,1023,0.016074666132529575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,1023,0.016858667135238647
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,2047,0.017152000218629837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,2047,0.01926400015751521
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,4095,0.02441066751877467
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,4095,0.02409599969784419
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,8191,0.0443200021982193
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,8191,0.03604800005753835
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,16383,0.06225066880385081
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,16383,0.05890133480230967
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,1,0.011952000359694162
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,1,0.013653332988421122
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,3,0.012058666596810022
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,3,0.01333333303531011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,7,0.012298667182525
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,7,0.013210666676362356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,63,0.012144000579913458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,63,0.013301332791646322
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,15,0.012175999581813812
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,15,0.013343999783198038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,31,0.01221866657336553
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,31,0.013178666432698568
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,127,0.012229333321253458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,127,0.01471466695268949
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,255,0.013408000270525614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,255,0.01498666654030482
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,511,0.015360000232855478
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,511,0.01643199970324834
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,1023,0.016176000237464905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,1023,0.016965333372354507
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,2047,0.018538666268189747
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,2047,0.019637333850065868
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,4095,0.03755199909210205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,4095,0.025301332275072735
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,8191,0.05820799867312113
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,8191,0.042725334564844765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,16383,0.1032373309135437
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,16383,0.06772266825040181
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,1,0.015146666516860327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,fp8,1,0.016106666376193363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,3,0.015066667149464289
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,fp8,3,0.016309333344300587
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,7,0.014949332922697067
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,fp8,15,0.01626666635274887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,fp8,7,0.01640533283352852
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,15,0.015072000523408255
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,31,0.015141333142916361
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,fp8,31,0.01613333324591319
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,63,0.015205333630243937
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,fp8,63,0.016309333344300587
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,127,0.015477333217859268
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,fp8,127,0.01748266691962878
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,fp8,255,0.01812800019979477
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,255,0.017786666750907898
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,511,0.02085866779088974
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,fp8,511,0.020986666282018025
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,1,0.018266666680574417
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,1023,0.03808533400297165
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,fp8,1023,0.027242665489514668
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,fp8,1,0.019978666057189304
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,3,0.018426666657129925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,fp8,3,0.01989866668979327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,7,0.01871466636657715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,fp8,7,0.019744000087181728
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,15,0.018474667022625606
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,fp8,31,0.020090666910012562
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,fp8,15,0.0199946661790212
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,31,0.018751999984184902
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,63,0.018474667022625606
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,fp8,63,0.020010666300853092
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,127,0.019050666441520054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,fp8,127,0.022842665513356526
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,255,0.022954667607943218
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,fp8,255,0.02421333392461141
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,511,0.04161066561937332
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,fp8,511,0.032511999209721885
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,1,0.025829332570234936
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,fp8,1,0.026890667776266735
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,1023,0.06347733239332835
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,fp8,1023,0.04846400022506714
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,3,0.025989333788553875
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,fp8,3,0.02733866622050603
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,7,0.025360000630219776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,fp8,7,0.027024000883102417
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,15,0.025194667279720306
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,fp8,31,0.02739199995994568
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,63,0.025439999997615814
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,fp8,15,0.026858667532602947
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,31,0.025455998877684276
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,fp8,63,0.02733866622050603
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,127,0.027248000105222065
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,fp8,127,0.031701333820819855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,255,0.04379733403523763
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,fp8,255,0.03850133220354716
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,511,0.06607466439406078
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,fp8,511,0.05487999816735586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,1,0.037765334049860634
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,fp8,1,0.04138133426507314
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,3,0.03815466662247976
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,1023,0.10808533430099487
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,fp8,1023,0.07850133379300435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,fp8,3,0.04074133435885111
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,fp8,7,0.04098666707674662
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,7,0.03801066676775614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,15,0.03841600070397059
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,fp8,15,0.041477332512537636
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,31,0.03877866764863332
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,fp8,63,0.04102933406829834
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,127,0.048357332746187844
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,fp8,127,0.05514133473237356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,255,0.06929066777229309
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,fp8,255,0.06451199948787689
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,63,0.04019733270009359
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,fp8,31,0.04162133236726125
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,1023,0.19529600938161215
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,fp8,1023,0.13491732875506082
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,511,0.11217600107192993
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,fp8,511,0.08787733316421509
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,1,0.022277332842350006
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,fp8,1,0.02386133372783661
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,3,0.02233600119749705
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,fp8,3,0.023919999599456787
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,fp8,7,0.02407466620206833
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,7,0.022175999979178112
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,31,0.02232533444960912
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,fp8,31,0.024197332561016083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,15,0.022304000953833263
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,63,0.02237333357334137
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,fp8,15,0.02359466751416524
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,fp8,63,0.024122667809327442
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,127,0.02345066765944163
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,fp8,127,0.02632533262173335
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,255,0.028933333853880566
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,1,0.02834133307139079
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,fp8,255,0.028501334289709728
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,fp8,1,0.030400000512599945
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,3,0.028410665690898895
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,7,0.028592000404993694
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,fp8,3,0.030618667602539062
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,fp8,7,0.03028800090154012
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,15,0.028751999139785767
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,fp8,15,0.030576000610987347
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,31,0.02903999884923299
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,fp8,31,0.030597334106763203
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,63,0.02863466739654541
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,fp8,63,0.030949334303538006
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,127,0.033557333052158356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,fp8,127,0.035317334036032356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,255,0.04896000027656555
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,fp8,255,0.04275199770927429
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,1,0.04180799921353658
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,fp8,1,0.04460266729195913
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,3,0.041349334021409355
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,fp8,3,0.04523199796676636
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,7,0.04156800111134847
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,fp8,7,0.04424533247947693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,31,0.04230933388074239
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,15,0.0417546679576238
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,fp8,15,0.04483733574549357
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,fp8,31,0.044794668753941856
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,fp8,63,0.04565866788228353
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,63,0.045834665497144066
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,127,0.0524479995171229
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,fp8,127,0.05718400080998739
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,1,0.06746666630109151
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,255,0.07375466823577881
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,fp8,255,0.06901333232720692
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,fp8,1,0.07152533531188965
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,3,0.06717333197593689
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,fp8,3,0.07145066559314728
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,7,0.06735999882221222
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,fp8,7,0.07129066685835521
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,15,0.06770666440327962
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,fp8,31,0.07167466481526692
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,fp8,15,0.07247466842333476
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,31,0.07145600020885468
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,63,0.07392000158627827
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,fp8,63,0.07626666625340779
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,fp8,127,0.09620799620946248
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,127,0.08158400158087413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,255,0.12079466382662456
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,fp8,255,0.11097600062688191
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,1,0.04095466683308283
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,fp8,1,0.04264533519744873
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,3,0.041333332657814026
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,fp8,3,0.04247466723124186
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,7,0.040591999888420105
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,fp8,7,0.04225599765777588
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,fp8,15,0.04252266883850098
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,15,0.04138666639725367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,31,0.04121600091457367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,fp8,31,0.042725334564844765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,63,0.041493333876132965
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,fp8,63,0.04264000058174133
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,127,0.04882133503754934
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,fp8,127,0.04684799909591675
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,1,0.05288533369700114
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,fp8,3,0.05570666491985321
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,3,0.052757332722345986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,7,0.05230933427810669
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,fp8,7,0.05473066866397858
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,15,0.05248000224431356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,fp8,15,0.055445333321889244
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,31,0.05353599786758423
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,fp8,31,0.05553600192070007
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,63,0.059088001648585
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,fp8,63,0.055162668228149414
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,127,0.06656000018119812
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,fp8,127,0.06938133140405019
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,1,0.07904000083605449
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,fp8,1,0.0846613347530365
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,3,0.07925333579381307
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,fp8,3,0.08425066868464152
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,7,0.07846400141716003
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,fp8,7,0.08368000388145447
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,15,0.0796319991350174
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,fp8,15,0.08402132987976074
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,31,0.08753599723180135
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,fp8,31,0.08502399921417236
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,fp8,63,0.09062932928403218
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,63,0.08830400307973225
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,fp8,1,0.05570666491985321
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,1,0.13086400429407755
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,fp8,127,0.10728533069292705
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,127,0.09488532940546672
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,fp8,1,0.13871467113494873
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,3,0.13142399986584982
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,fp8,3,0.1368106702963511
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,fp8,7,0.13852799932161966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,15,0.1383573313554128
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,7,0.13089600205421448
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,fp8,15,0.13758933544158936
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,31,0.13891200224558511
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,fp8,31,0.1455519994099935
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,1,0.011578666667143503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,1,0.012335999558369318
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,63,0.141594668229421
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,7,0.011429333438475927
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,3,0.01126933346192042
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,7,0.012309333930412928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,3,0.012533333152532578
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,15,0.01250133290886879
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,31,0.011226666470368704
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,31,0.012479999413092932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,15,0.011525332927703857
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,63,0.011605333536863327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,63,0.01259200026591619
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,127,0.011359999577204386
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,127,0.013568000247081121
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,fp8,63,0.1453386644522349
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,255,0.012527999778588613
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,255,0.015008000036080679
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,127,0.15341333548227945
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,511,0.01440000037352244
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,511,0.01569066693385442
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,1023,0.016517333686351776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,fp8,127,0.1797013282775879
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,4095,0.03311466674009959
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,2047,0.020869334538777668
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,2047,0.023285334308942158
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,1023,0.018346666047970455
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,8191,0.04785599807898203
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,8191,0.05207466582457224
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,1,0.012800000607967377
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,4095,0.02956799914439519
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,16383,0.09502933422724406
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,1,0.0116799995303154
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,3,0.011567999919255575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,16383,0.08397333820660909
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,3,0.01257066677014033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,7,0.0116799995303154
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,7,0.012826666235923767
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,15,0.011754666765530905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,15,0.012949333836634954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,31,0.011637333780527115
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,31,0.012650666137536367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,63,0.011882666498422623
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,63,0.012634667257467905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,127,0.011823999385039011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,127,0.01402666668097178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,255,0.013173333058754602
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,255,0.01586666703224182
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,511,0.014767999450365702
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,511,0.016879999389251072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,1023,0.01720000058412552
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,1023,0.019199999670187633
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,2047,0.021557333568731945
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,2047,0.023706667125225067
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,4095,0.030778666337331135
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,4095,0.03329066683848699
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,8191,0.04959466556708018
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,8191,0.05417066812515259
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,16383,0.08684266606966655
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,16383,0.10050666332244873
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,1,0.01210133358836174
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,1,0.01339200014869372
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,3,0.012229333321253458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,3,0.01331199953953425
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,7,0.011930666863918304
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,7,0.013178666432698568
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,15,0.012565333396196365
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,15,0.013690666606028875
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,31,0.012346666306257248
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,31,0.01333333303531011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,63,0.012389333297808966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,63,0.013525333255529404
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,127,0.012560000022252401
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,127,0.014767999450365702
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,255,0.013642666240533194
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,255,0.015930666277805965
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,511,0.015344000111023584
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,511,0.016544000556071598
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,1023,0.01693333312869072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,1023,0.01926400015751521
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,4095,0.035301332672437034
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,2047,0.0225600004196167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,2047,0.02420266717672348
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,16383,0.10173333684603374
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,4095,0.04223999877770742
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,8191,0.061146666606267296
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,3,0.012319999436537424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,8191,0.0584746648867925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,16383,0.10703999797503154
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,1,0.011946666985750198
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,1,0.013807999591032663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,3,0.01358933374285698
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,7,0.01201066623131434
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,7,0.013306666165590286
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,15,0.012266666938861212
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,15,0.013455999394257864
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,31,0.012357333054145178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,31,0.01322666679819425
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,63,0.012485332787036896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,63,0.013818666338920593
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,127,0.01227733368674914
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,127,0.014938666174809137
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,255,0.013962666193644205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,255,0.01639466608564059
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,511,0.015642666568358738
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,511,0.017194667210181553
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,1023,0.018581333259741466
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,1023,0.01941866676012675
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,2047,0.03733866661787033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,2047,0.025914666553338368
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,4095,0.058917333682378135
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,4095,0.043178667624791466
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,8191,0.102101335922877
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,8191,0.06624533236026764
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,16383,0.1896053353945414
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,16383,0.11466667056083679
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,1,0.03923200070858002
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,fp8,1,0.03902933249870936
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,fp8,3,0.039647998909155525
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,7,0.038975998759269714
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,3,0.039034667114416756
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,fp8,7,0.03904533386230469
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,fp8,15,0.03956266740957896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,15,0.04042666653792063
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,31,0.04387733340263367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,fp8,31,0.03951466580231985
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,63,0.047024001677831016
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,fp8,63,0.04107200105985006
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,fp8,1,0.06323733429114024
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,1,0.06498133142789204
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,3,0.06564799944559734
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,fp8,3,0.06377066671848297
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,7,0.06672533353169759
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,fp8,7,0.06307733555634816
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,15,0.0708426684141159
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,fp8,15,0.06366399923960368
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,31,0.07316799958546956
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,fp8,31,0.06761066615581512
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,63,0.07494399944941203
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,fp8,63,0.07358933488527934
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,1,0.11667733391125996
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,fp8,1,0.11366933584213257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,3,0.11799466609954834
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,fp8,3,0.11346667011578877
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,fp8,7,0.11477866768836975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,7,0.12376532951990764
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,15,0.1295093297958374
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,fp8,15,0.1181066632270813
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,fp8,31,0.1260479986667633
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,31,0.1291253368059794
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,63,0.13081600268681845
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,fp8,1,0.20632533232371011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,fp8,63,0.12866133451461792
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,1,0.21402132511138916
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,3,0.22689066330591837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,fp8,3,0.2065920035044352
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,7,0.23043199380238852
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,fp8,7,0.21336533625920615
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,15,0.23079999287923178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,fp8,15,0.22226667404174805
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,31,0.23365867137908936
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,fp8,31,0.22842133045196533
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,63,0.2366080085436503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,fp8,63,0.22592000166575113
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,float16,1,0.07773333291212718
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,fp8,1,0.06959466636180878
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,fp8,3,0.06989866495132446
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,float16,3,0.07572799921035767
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,float16,7,0.07753600180149078
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,fp8,7,0.07390933235486348
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,fp8,15,0.07565866907437642
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,float16,15,0.07814399898052216
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,float16,31,0.07778133451938629
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,fp8,31,0.07902400195598602
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,float16,1,0.13158399860064188
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,fp8,1,0.11901332934697469
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,float16,3,0.13222933808962503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,fp8,3,0.12107732892036438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,float16,7,0.1311840017636617
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,fp8,7,0.12431466579437256
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,float16,15,0.13218133648236594
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,fp8,15,0.12705066800117493
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,float16,31,0.1327946682771047
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,float16,1,0.24333866437276205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,fp8,31,0.1288800040880839
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,fp8,1,0.2271946668624878
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,float16,3,0.2421813408533732
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,fp8,3,0.23105067014694214
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,float16,7,0.24227199951807657
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,fp8,7,0.23161067565282187
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,float16,15,0.24378132820129395
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,fp8,15,0.2355253299077352
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,float16,31,0.24329066276550293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,float16,1,0.44657599925994873
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,fp8,1,0.42767465114593506
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,fp8,31,0.23603200912475586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,float16,3,0.4470826784769694
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,1,0.012015999605258306
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,fp8,1,0.012794667234023413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,fp8,3,0.42587733268737793
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,3,0.011898666620254517
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,float16,7,0.44863466421763104
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,fp8,3,0.012746666868527731
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,7,0.011760000139474869
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,fp8,7,0.429749329884847
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,15,0.011541333049535751
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,fp8,7,0.013002666334311167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,fp8,15,0.012842666357755661
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,float16,15,0.44795199235280353
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,31,0.011578666667143503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,63,0.01198400060335795
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,fp8,63,0.013013333082199097
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,fp8,31,0.013007999708255133
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,127,0.011546666423479715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,fp8,127,0.014159999787807465
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,fp8,255,0.014842666685581207
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,fp8,15,0.42871999740600586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,511,0.015034666905800501
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,255,0.013002666334311167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,fp8,511,0.017269333203633625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,1023,0.019802667200565338
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,fp8,1023,0.022287999590237934
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,2047,0.02812266598145167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,4095,0.046341334780057274
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,fp8,4095,0.051029334465662636
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,fp8,2047,0.03198933353026708
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,8191,0.08251733581225078
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,fp8,8191,0.09240000446637471
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,1,0.012170666207869848
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,fp8,1,0.013194666554530462
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,fp8,31,0.43029868602752686
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,3,0.011893333246310553
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,fp8,3,0.013258667041858038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,7,0.011941333611806234
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,fp8,7,0.013242666920026144
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,15,0.012213333199421564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,fp8,15,0.013280000537633896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,31,0.011823999385039011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,63,0.011893333246310553
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,fp8,63,0.013936000565687815
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,fp8,31,0.01351999988158544
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,127,0.012549333274364471
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,fp8,127,0.01444799949725469
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,255,0.013877333452304205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,fp8,255,0.015311999867359797
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,511,0.01562133307258288
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,fp8,511,0.018357332795858383
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,1023,0.01977066695690155
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,fp8,1023,0.022405333817005157
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,2047,0.030005333324273426
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,fp8,2047,0.03281066566705704
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,4095,0.049685334165891014
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,fp8,4095,0.053786665201187134
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,8191,0.0872213343779246
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,fp8,8191,0.09916266798973083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,1,0.012106666962305704
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,fp8,1,0.013637332866589228
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,3,0.012421333541472753
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,fp8,3,0.013893333574136099
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,7,0.012319999436537424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,fp8,7,0.013424000392357508
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,31,0.01258133351802826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,15,0.012106666962305704
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,fp8,15,0.013514666507641474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,fp8,31,0.013882666826248169
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,63,0.012645332763592402
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,fp8,63,0.013445333888133367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,127,0.012629333883523941
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,float16,31,0.4530506531397502
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,fp8,127,0.01498666654030482
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,255,0.01393066719174385
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,fp8,255,0.016037333756685257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,511,0.016255999604860943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,fp8,511,0.018351999421914417
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,1023,0.021530665457248688
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,fp8,1023,0.02258666604757309
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,fp8,2047,0.03469333300987879
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,4095,0.0595413347085317
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,fp8,4095,0.0580266664425532
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,2047,0.036559998989105225
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,8191,0.10437867045402527
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,fp8,8191,0.09940266609191895
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,1,0.01240533341964086
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,fp8,1,0.013610667238632837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,3,0.01246400053302447
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,fp8,3,0.013605333864688873
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,7,0.012362666428089142
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,fp8,7,0.013621332744757334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,15,0.012671999633312225
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,fp8,15,0.013637332866589228
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,31,0.012543999900420507
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,fp8,31,0.01402666668097178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,63,0.01259200026591619
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,fp8,63,0.014042666802803675
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,127,0.013023999830087027
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,fp8,127,0.015253332753976187
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,255,0.014773332824309668
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,fp8,255,0.015824000040690105
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,511,0.017152000218629837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,fp8,511,0.018757333358128864
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,1023,0.035504000882307686
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,fp8,1023,0.024666666984558105
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,2047,0.056976000467936196
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,fp8,2047,0.04114133367935816
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,4095,0.10098133484522502
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,1,0.012389333297808966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,fp8,4095,0.0655626654624939
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,8191,0.18837332725524902
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,fp8,8191,0.11293333768844604
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,fp8,1,0.013301332791646322
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,3,0.012330666184425354
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,fp8,3,0.012997332960367203
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,7,0.011882666498422623
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,fp8,7,0.01321600005030632
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,15,0.012330666184425354
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,fp8,15,0.01302933320403099
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,31,0.012085333466529846
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,fp8,31,0.013365333278973898
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,63,0.012138667205969492
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,fp8,63,0.013093333691358566
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,127,0.01268799975514412
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,fp8,511,0.018053332964579265
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,fp8,127,0.014826666563749313
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,255,0.013552000125249227
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,fp8,255,0.015439999600251516
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,511,0.01543466622630755
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,2047,0.030207999050617218
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,1023,0.020282667130231857
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,fp8,1023,0.022202665607134502
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,fp8,2047,0.03294933338960012
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,4095,0.050437331199645996
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,fp8,4095,0.05353599786758423
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,8191,0.08457066615422566
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,fp8,8191,0.09962133566538493
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,1,0.012543999900420507
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,fp8,1,0.013210666676362356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,3,0.012586666891972223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,fp8,3,0.013786666095256805
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,7,0.01202133297920227
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,fp8,7,0.013503999759753546
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,fp8,31,0.014848000059525171
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,15,0.012042666474978128
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,fp8,15,0.013855999956528345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,31,0.01246400053302447
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,63,0.01267733300725619
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,fp8,63,0.013818666338920593
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,127,0.012389333297808966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,fp8,127,0.015216000378131866
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,255,0.013834666460752487
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,fp8,255,0.015642666568358738
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,511,0.01597333326935768
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,fp8,511,0.018735999862353008
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,1023,0.02086399992307027
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,fp8,1023,0.023029332359631855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,2047,0.037258667250474296
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,fp8,2047,0.03402133285999298
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,4095,0.06945066650708516
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,fp8,4095,0.057962665955225624
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,8191,0.1053653359413147
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,fp8,8191,0.09988799691200256
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,1,0.012613333761692047
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,fp8,1,0.013754667093356451
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,3,0.012997332960367203
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,fp8,3,0.013877333452304205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,7,0.012746666868527731
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,fp8,7,0.013994666437307993
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,15,0.012986666212479273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,fp8,15,0.013653332988421122
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,31,0.012613333761692047
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,fp8,31,0.013653332988421122
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,63,0.012928000340859095
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,fp8,63,0.014138666292031607
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,127,0.01313599944114685
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,fp8,127,0.015413332730531693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,255,0.014416000495354334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,fp8,255,0.01579733317097028
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,511,0.01749333366751671
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,fp8,511,0.01854933301607768
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,1023,0.03544000039498011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,fp8,1023,0.023818666736284893
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,2047,0.05780800183614095
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,fp8,2047,0.041296000281969704
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,4095,0.10125333070755005
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,fp8,4095,0.06588266789913177
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,8191,0.18829333782196045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,fp8,8191,0.11222400267918904
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,1,0.016757333030303318
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,fp8,1,0.018298666924238205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,3,0.016597333053747814
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,15,0.01670933390657107
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,fp8,3,0.017952000101407368
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,7,0.016389333953460056
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,fp8,7,0.01791999985774358
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,63,0.016917333006858826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,fp8,15,0.0179626668492953
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,31,0.016714667280515034
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,fp8,31,0.018351999421914417
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,fp8,63,0.018031999468803406
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,127,0.01711999997496605
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,fp8,127,0.020645332833131153
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,255,0.02054399996995926
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,fp8,255,0.022474666436513264
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,511,0.03752533346414566
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,2047,0.10316266616185506
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,fp8,511,0.03035199890534083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,1023,0.05927466849486033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,4095,0.18957332770029703
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,fp8,1023,0.044490665197372437
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,fp8,2047,0.06930133203665416
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,fp8,4095,0.11731732885042827
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,fp8,8191,0.2108746568361918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,1,0.011503999431928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,3,0.012437333663304647
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,1,0.012752000242471695
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,3,0.011488000551859537
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,7,0.011370666325092316
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,8191,0.36452265580495197
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,7,0.012703999876976013
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,15,0.012495999534924826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,15,0.012863999853531519
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,31,0.011525332927703857
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,31,0.012719999998807907
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,63,0.01145600030819575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,63,0.01246400053302447
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,127,0.011359999577204386
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,127,0.013653332988421122
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,255,0.012629333883523941
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,255,0.013866666704416275
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,511,0.012527999778588613
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,511,0.014352000008026758
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,1023,0.013104000439246496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,1023,0.014282666146755219
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,2047,0.014346666634082794
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,2047,0.01575999955336253
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,4095,0.015775999675194424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,4095,0.016672000288963318
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,8191,0.01781333362062772
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,8191,0.01971199984351794
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,16383,0.022330666581789654
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,16383,0.024677333732446034
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,32767,0.03234666585922241
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,32767,0.035061334570248924
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,1,0.011498666057984034
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,1,0.012400000045696894
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,15,0.011381333072980246
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,3,0.011413333316644033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,3,0.012389333297808966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,7,0.01128000020980835
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,63,0.011541333049535751
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,7,0.01303999995191892
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,15,0.012960000584522883
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,31,0.01138666644692421
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,31,0.012944000462690989
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,63,0.012330666184425354
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,127,0.01126933346192042
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,127,0.013994666437307993
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,255,0.012469333906968435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,255,0.013829333086808523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,511,0.012847999731699625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,511,0.01413333291808764
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,1023,0.012975999464591345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,1023,0.014783999572197596
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,2047,0.014741333822409311
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,2047,0.01621333385507266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,4095,0.016373333831628162
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,4095,0.017749333133300144
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,8191,0.01886933296918869
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,8191,0.020506666352351505
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,16383,0.023434666295846302
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,1,0.01173866664369901
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,16383,0.026127999027570088
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,32767,0.033333333830038704
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,32767,0.03549866626660029
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,1,0.012527999778588613
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,3,0.011600000162919363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,3,0.012159999459981918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,7,0.011685332904259363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,7,0.012725333372751871
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,15,0.011776000261306763
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,15,0.012602667013804117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,31,0.011642667154471079
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,31,0.012495999534924826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,63,0.011594666788975397
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,63,0.012426666915416718
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,127,0.011952000359694162
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,127,0.013557333499193192
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,255,0.01312000056107839
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,255,0.013754667093356451
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,511,0.012970666090647379
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,511,0.014346666634082794
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,1023,0.014106666048367819
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,1023,0.01617066686352094
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,2047,0.015765332927306492
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,2047,0.016677333662907284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,4095,0.016832000265518825
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,4095,0.017717332889636356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,8191,0.019071999937295914
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,8191,0.02038399999340375
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,16383,0.024357333779335022
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,16383,0.02622933437426885
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,32767,0.039018665750821434
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,32767,0.03710933278004328
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,1,0.011823999385039011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,1,0.012682666381200155
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,3,0.011760000139474869
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,3,0.012458667159080505
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,7,0.011701333026091257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,7,0.01309866706530253
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,15,0.011776000261306763
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,15,0.012826666235923767
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,31,0.01190399999419848
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,31,0.012757333616415659
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,63,0.011749333391586939
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,63,0.012527999778588613
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,127,0.011663999408483505
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,127,0.013914667069911957
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,255,0.012970666090647379
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,255,0.014602666099866232
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,511,0.0138026662170887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,511,0.014933332800865173
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,1023,0.013957332819700241
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,1023,0.015263999501864115
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,2047,0.01552533358335495
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,2047,0.01721599946419398
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,4095,0.016832000265518825
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,4095,0.01834133391578992
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,8191,0.020586666961510975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,8191,0.02089600016673406
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,16383,0.037621334195137024
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,16383,0.027552001178264618
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,32767,0.06080000102519989
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,32767,0.04324266811211904
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,1,0.011343999455372492
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,1,0.012810666114091873
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,3,0.011461333682139715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,3,0.012773333738247553
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,7,0.011328000575304031
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,7,0.01227733368674914
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,15,0.011557333171367645
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,15,0.012874666601419449
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,31,0.011343999455372492
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,31,0.012730666746695837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,63,0.011477333803971609
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,63,0.012533333152532578
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,127,0.011418666690587997
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,127,0.013754667093356451
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,255,0.012901333471139273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,255,0.013872000078360239
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,511,0.013034666577974955
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,511,0.014282666146755219
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,1023,0.01403733342885971
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,1023,0.015098666151364645
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,2047,0.015333333363135656
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,2047,0.016629333297411602
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,4095,0.017829333742459614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,4095,0.01907733331123988
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,8191,0.021168000996112823
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,8191,0.02367466688156128
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,16383,0.029711998999118805
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,16383,0.03422933320204417
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,32767,0.048170665899912514
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,32767,0.05286400020122528
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,1,0.011733333269755045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,7,0.013408000270525614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,1,0.012842666357755661
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,3,0.011600000162919363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,3,0.012298667182525
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,7,0.011600000162919363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,15,0.011482667177915573
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,15,0.012752000242471695
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,31,0.011637333780527115
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,31,0.012815999488035837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,63,0.011557333171367645
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,255,0.012970666090647379
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,63,0.012576000144084295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,127,0.011482667177915573
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,127,0.01381333296497663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,255,0.014256000518798828
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,511,0.013258667041858038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,511,0.01488000030318896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,1023,0.014592000593741735
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,1023,0.016208000481128693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,2047,0.016234666109085083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,2047,0.016837333639462788
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,4095,0.018378666291634243
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,4095,0.019861333072185516
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,8191,0.022042666872342426
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,8191,0.02458133300145467
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,16383,0.03153600047032038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,16383,0.0341386670867602
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,32767,0.050426666935284935
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,32767,0.05504000186920166
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,1,0.01181866725285848
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,1,0.012885333349307379
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,3,0.011823999385039011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,3,0.012543999900420507
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,7,0.011855999628702799
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,7,0.012693333129088083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,15,0.011653333902359009
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,15,0.01268799975514412
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,31,0.01191466674208641
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,31,0.013013333082199097
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,63,0.011792000383138657
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,63,0.01250133290886879
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,127,0.011957333733638128
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,511,0.015413332730531693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,127,0.013914667069911957
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,255,0.012970666090647379
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,255,0.014783999572197596
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,4095,0.01904533306757609
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,511,0.014378666877746582
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,4095,0.019466667125622433
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,8191,0.02309866746266683
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,1023,0.015610666324694952
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,1023,0.015909332782030106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,2047,0.016783999900023144
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,2047,0.017429333180189133
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,8191,0.024101334313551586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,16383,0.039103999733924866
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,16383,0.03595733394225439
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,32767,0.06276800235112508
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,32767,0.05778666834036509
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,1,0.011909333368142446
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,1,0.012954667210578918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,3,0.012165332833925882
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,3,0.013173333058754602
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,7,0.012096000214417776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,7,0.012874666601419449
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,15,0.012015999605258306
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,15,0.013167999684810638
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,31,0.011834666132926941
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,31,0.013088000317414602
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,63,0.011968000481526056
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,63,0.013248000293970108
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,127,0.012058666596810022
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,127,0.01471466695268949
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,255,0.013610667238632837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,255,0.01492799942692121
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,511,0.013781332721312841
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,2047,0.01748266691962878
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,511,0.015471999843915304
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,4095,0.020202666521072388
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,1023,0.015749332805474598
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,1023,0.016773333152135212
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,2047,0.016885332763195038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,4095,0.019434666881958645
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,8191,0.03692266593376795
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,8191,0.02589866767326991
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,32767,0.07554666697978973
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,16383,0.05862933397293091
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,16383,0.04608533283074697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,32767,0.10311466455459595
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,1,0.01312000056107839
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,fp8,1,0.01368533323208491
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,fp8,3,0.01368533323208491
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,3,0.013007999708255133
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,7,0.012826666235923767
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,fp8,7,0.013679999858140945
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,15,0.01302933320403099
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,fp8,15,0.013786666095256805
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,31,0.012981332838535309
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,fp8,31,0.014181333283583323
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,63,0.013167999684810638
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,fp8,63,0.013408000270525614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,127,0.013621332744757334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,fp8,127,0.015024000157912573
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,255,0.015040000279744467
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,fp8,255,0.016255999604860943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,511,0.017093333105246227
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,fp8,511,0.01907733331123988
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,1023,0.024432001014550526
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,fp8,1023,0.023285334308942158
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,2047,0.03811199963092804
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,fp8,2047,0.03467733412981033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,4095,0.061610668897628784
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,1,0.013210666676362356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,fp8,4095,0.05862933397293091
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,fp8,1,0.014416000495354334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,3,0.013525333255529404
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,fp8,3,0.014303999642531076
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,7,0.013477332890033722
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,fp8,7,0.01471466695268949
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,15,0.01332266628742218
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,fp8,15,0.014597332725922266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,fp8,31,0.0145066666106383
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,31,0.013359999905029932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,63,0.013365333278973898
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,fp8,63,0.014282666146755219
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,127,0.013877333452304205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,fp8,127,0.015802666544914246
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,255,0.015173333386580149
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,fp8,255,0.01648533344268799
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,511,0.017968000223239262
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,fp8,511,0.01942933350801468
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,1023,0.036159999668598175
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,fp8,1023,0.02497600018978119
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,2047,0.058149332801500954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,1,0.0170666662355264
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,4095,0.10196266571680705
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,fp8,4095,0.06610666712125142
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,fp8,2047,0.04240000247955322
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,fp8,1,0.018021332720915478
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,3,0.01749333366751671
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,fp8,7,0.017903999735911686
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,fp8,15,0.0179626668492953
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,fp8,3,0.017637333522240322
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,7,0.017418666432301205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,15,0.017301333447297413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,fp8,31,0.01803733284274737
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,31,0.01722666621208191
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,63,0.01748266691962878
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,fp8,63,0.017850667238235474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,127,0.017637333522240322
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,fp8,127,0.020661332954963047
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,255,0.023221333821614582
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,fp8,255,0.02223466585079829
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,511,0.03949866692225138
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,fp8,511,0.02834133307139079
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,1023,0.06038400034109751
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,2047,0.10353066523869832
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,fp8,1023,0.045082668463389076
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,fp8,2047,0.06981333096822102
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,4095,0.1905333399772644
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,fp8,4095,0.1167093316713969
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,1,0.024122667809327442
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,fp8,1,0.025946666797002155
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,fp8,3,0.025765334566434223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,3,0.02383466561635335
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,fp8,7,0.025248001019159954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,15,0.024165332317352295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,fp8,15,0.02536533276240031
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,7,0.02382933348417282
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,31,0.024373332659403484
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,fp8,63,0.02604266752799352
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,63,0.02456533412138621
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,fp8,31,0.025754667818546295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,fp8,127,0.030389333764712017
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,127,0.025120000044504803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,255,0.04266133407751719
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,fp8,255,0.03500800083080927
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,1023,0.10635200142860413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,fp8,1023,0.07517333328723907
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,fp8,511,0.051594664653142296
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,511,0.0635040005048116
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,2047,0.19034665822982788
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,1,0.011247999966144562
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,fp8,2047,0.12355200449625652
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,fp8,4095,0.24010133743286133
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,3,0.01180800050497055
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,4095,0.3631146748860677
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,7,0.011226666470368704
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,7,0.011920000116030375
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,1,0.012842666357755661
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,3,0.010863999525705973
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,15,0.01102399950226148
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,15,0.011834666132926941
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,31,0.010938666760921478
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,31,0.012389333297808966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,63,0.011018666128317514
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,63,0.012778667112191519
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,127,0.010911999891201654
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,127,0.013525333255529404
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,255,0.012357333054145178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,255,0.013557333499193192
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,511,0.013861333330472311
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,511,0.014767999450365702
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,1023,0.014831999937693277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,1023,0.016330666840076447
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,2047,0.016421332955360413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,2047,0.01842133328318596
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,4095,0.02027200038234393
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,4095,0.022890667120615642
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,8191,0.029050665597120922
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,8191,0.03294933338960012
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,16383,0.047397335370381675
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,16383,0.0524586687485377
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,32767,0.08406399687131245
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,32767,0.09657599528630574
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,1,0.011183999478816986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,1,0.012538666526476542
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,3,0.011296000331640244
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,3,0.01209066684047381
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,7,0.01139733319481214
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,7,0.012128000458081564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,15,0.011322667201360067
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,15,0.01210133358836174
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,31,0.011231999844312668
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,31,0.01250133290886879
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,63,0.011226666470368704
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,63,0.012175999581813812
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,127,0.011445333560307821
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,127,0.013541333377361298
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,255,0.012938667088747025
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,255,0.014570667097965876
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,511,0.014149333039919535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,511,0.015930666277805965
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,1023,0.015978666643301647
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,1023,0.016800000021855038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,2047,0.017477333545684814
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,2047,0.019023999571800232
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,4095,0.02144533395767212
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,4095,0.023562667270501454
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,8191,0.029845332105954487
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,8191,0.03417599946260452
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,16383,0.04927466809749603
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,16383,0.05402666827042898
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,32767,0.0849173367023468
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,32767,0.10090667009353638
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,7,0.011882666498422623
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,1,0.011685332904259363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,1,0.012047999848922094
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,3,0.011749333391586939
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,3,0.01259200026591619
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,7,0.0122079998254776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,15,0.01201066623131434
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,15,0.012282667060693106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,31,0.011829332758982977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,31,0.012416000167528788
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,63,0.0116799995303154
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,63,0.012367999802033106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,127,0.011936000237862269
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,127,0.014479999740918478
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,255,0.014127999544143677
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,255,0.0141546664138635
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,511,0.015103999525308609
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,511,0.015802666544914246
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,1023,0.01579733317097028
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,1023,0.01669866715868314
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,2047,0.017680000513792038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,2047,0.018816000471512478
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,16383,0.06171200176080068
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,4095,0.023754666248957317
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,4095,0.023962666591008503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,8191,0.03818133225043615
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,8191,0.03399466723203659
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,16383,0.054666668176651
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,32767,0.11377599835395813
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,3,0.013679999858140945
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,32767,0.09582400321960449
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,1,0.01219733307758967
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,1,0.013744000345468521
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,15,0.011962667107582092
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,3,0.011999999483426413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,7,0.012026666353146235
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,7,0.013338666409254074
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,15,0.013349333157142004
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,31,0.011786667009194693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,31,0.0138026662170887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,63,0.012426666915416718
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,63,0.013317332913478216
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,127,0.012362666428089142
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,127,0.014554666976133982
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,255,0.013770667215188345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,255,0.014826666563749313
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,511,0.015376000354687372
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,511,0.016496000190575916
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,1023,0.01642666632930438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,4095,0.036176001032193504
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,4095,0.026378666361172993
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,1023,0.017125333348910015
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,2047,0.018789333601792652
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,2047,0.01956266661485036
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,8191,0.05874133110046387
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,8191,0.04680533210436503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,16383,0.10172266761461894
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,16383,0.06670933465162913
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,32767,0.18997865915298462
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,32767,0.13134933511416116
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,1,0.014565333724021912
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,fp8,1,0.016106666376193363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,3,0.014762666076421738
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,fp8,3,0.015674666812022526
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,7,0.01479999969402949
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,fp8,7,0.016010666886965435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,15,0.014485333114862442
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,fp8,15,0.015770666301250458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,31,0.014645333091417948
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,fp8,31,0.015893333901961643
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,63,0.01452800010641416
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,fp8,63,0.015893333901961643
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,127,0.01573866605758667
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,fp8,127,0.017173333714405697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,255,0.01655999943614006
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,fp8,255,0.018031999468803406
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,511,0.020773333807786305
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,fp8,511,0.020474666108687718
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,1023,0.03796799977620443
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,fp8,1023,0.026357332865397137
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,1,0.017973333597183228
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,fp8,1,0.019717333217461903
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,3,0.017674667139848072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,2047,0.06026133398214976
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,fp8,2047,0.04438933233420054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,fp8,3,0.01940800001223882
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,7,0.01802666609485944
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,fp8,7,0.01947733387351036
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,15,0.01820266619324684
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,fp8,15,0.019600000232458115
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,31,0.01781333362062772
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,fp8,31,0.02013333390156428
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,fp8,63,0.019487999379634857
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,63,0.01815466706951459
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,127,0.01836266616980235
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,fp8,127,0.02319466571013133
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,255,0.022757334013779957
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,fp8,255,0.023685333629449207
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,511,0.04089066634575526
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,fp8,511,0.031658666829268135
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,1023,0.06237333516279856
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,fp8,1023,0.047338664531707764
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,1,0.024714666108290356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,fp8,1,0.02645866572856903
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,3,0.024336000283559162
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,fp8,3,0.026149332523345947
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,2047,0.10517866412798564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,fp8,2047,0.07339199880758922
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,7,0.024490666886170704
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,fp8,7,0.026122666895389557
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,15,0.024720000723997753
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,fp8,15,0.026320000489552815
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,31,0.024613333245118458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,fp8,31,0.026288000245889027
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,127,0.026591998835404713
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,fp8,127,0.030960001051425934
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,255,0.04295999805132548
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,fp8,255,0.03640533238649368
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,511,0.06367466847101848
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,63,0.025114665428797405
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,fp8,511,0.05249066650867462
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,fp8,63,0.026634665826956432
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,fp8,2047,0.12476799885431926
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,2047,0.19326400756835938
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,1023,0.10619733730951945
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,fp8,1023,0.07610666751861572
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,fp8,1,0.041493333876132965
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,1,0.03730666637420654
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,fp8,3,0.04094400008519491
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,3,0.037615999579429626
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,7,0.03759466608365377
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,fp8,7,0.04075733323891958
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,31,0.038021333515644073
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,fp8,31,0.04101866732041041
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,15,0.03764266769091288
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,63,0.041082667807737984
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,fp8,63,0.040949332217375435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,fp8,15,0.040778666734695435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,127,0.0472320020198822
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,fp8,127,0.05402666827042898
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,255,0.07087466617425282
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,fp8,255,0.06302933394908905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,1023,0.1965386668841044
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,fp8,1023,0.1330560048421224
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,511,0.11044800281524658
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,fp8,511,0.08806932965914409
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,2047,0.3649386564890544
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,fp8,2047,0.23169066508611044
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,1,0.021781332790851593
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,fp8,1,0.023584000766277313
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,3,0.021941334009170532
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,15,0.02216000109910965
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,fp8,7,0.023946667710940044
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,fp8,3,0.023797333240509033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,fp8,15,0.02350933353106181
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,7,0.021941334009170532
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,31,0.022085333863894146
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,fp8,31,0.023743999501069386
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,63,0.021930667261282604
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,fp8,63,0.023978665471076965
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,127,0.022704000274340313
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,fp8,127,0.026378666361172993
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,255,0.027829334139823914
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,fp8,255,0.02805333336194356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,511,0.0458133320013682
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,1,0.027989332874615986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,fp8,511,0.0358240008354187
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,fp8,1,0.03054400036732356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,3,0.0281333327293396
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,fp8,3,0.030458666384220123
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,7,0.027952000498771667
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,fp8,7,0.030432000756263733
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,15,0.0284853329261144
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,fp8,15,0.030117332935333252
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,31,0.028575999041398365
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,fp8,31,0.030608000854651134
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,63,0.028624000648657482
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,fp8,63,0.030597334106763203
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,127,0.03145066648721695
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,fp8,127,0.03517866631348928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,255,0.047466665506362915
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,fp8,255,0.04041066765785217
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,1,0.041290665666262306
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,fp8,1,0.04327466587225596
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,511,0.07073066631952922
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,fp8,511,0.058335999647776283
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,3,0.04070399949947993
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,fp8,3,0.04363200068473816
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,7,0.041221333046754204
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,fp8,7,0.043061330914497375
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,15,0.04142933338880539
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,fp8,15,0.04337066908677419
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,63,0.04342400034268697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,fp8,63,0.04363200068473816
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,31,0.041434665520985924
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,127,0.0513866643110911
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,fp8,31,0.04383466641108195
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,fp8,127,0.05348266661167145
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,255,0.07107733190059662
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,fp8,255,0.06668266654014587
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,1,0.06684266527493794
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,fp8,1,0.07146133482456207
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,3,0.06726933519045512
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,511,0.1129813293615977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,fp8,511,0.09078400333722432
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,fp8,3,0.07110933462778728
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,7,0.06668800115585327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,fp8,7,0.07128533224264781
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,15,0.06719466547171275
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,fp8,15,0.07149333258469899
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,31,0.07108800113201141
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,fp8,63,0.07332799832026164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,127,0.08012266457080841
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,fp8,127,0.09400000174840291
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,63,0.07369066774845123
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,fp8,31,0.07120533287525177
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,fp8,255,0.10915199915568034
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,255,0.12212799986203511
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,1,0.03908266623814901
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,511,0.2058453361193339
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,fp8,511,0.15889599919319153
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,fp8,1,0.04112533231576284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,3,0.03931200007597605
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,fp8,3,0.040805332362651825
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,7,0.03912533322970072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,15,0.039279999832312264
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,fp8,7,0.041434665520985924
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,fp8,31,0.04165333261092504
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,63,0.039317332208156586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,fp8,63,0.04155733436346054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,fp8,15,0.04177600145339966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,31,0.03899200012286504
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,127,0.045696000258127846
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,fp8,127,0.045647998650868736
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,255,0.060677334666252136
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,fp8,255,0.0552106648683548
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,1,0.05070933202902476
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,fp8,1,0.054416000843048096
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,3,0.05052266518274943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,fp8,3,0.054042667150497437
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,7,0.0505920002857844
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,fp8,7,0.05468266705671946
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,15,0.05062933266162872
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,fp8,15,0.05386666456858317
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,31,0.05217066903909048
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,fp8,31,0.053898667295773826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,63,0.05596266686916351
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,fp8,63,0.05504000186920166
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,fp8,127,0.06763733426729839
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,127,0.06457599997520447
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,255,0.0846560001373291
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,fp8,255,0.08001066744327545
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,1,0.07594133416811626
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,fp8,1,0.08091199894746144
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,3,0.07749866445859273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,fp8,3,0.08098666866620381
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,7,0.07733333110809326
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,fp8,7,0.08081600069999695
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,15,0.07758933305740356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,fp8,31,0.08237866560618083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,31,0.08101333181063335
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,fp8,15,0.081535999973615
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,63,0.08353066444396973
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,fp8,63,0.08509332935015361
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,127,0.09154666463534038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,1,0.12879467010498047
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,fp8,1,0.13732799887657166
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,fp8,127,0.1039573351542155
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,3,0.12905066212018332
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,fp8,255,0.1202239990234375
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,255,0.12691733241081238
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,fp8,3,0.13532267014185587
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,7,0.129530668258667
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,fp8,7,0.135343998670578
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,15,0.13573867082595825
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,31,0.13384532928466797
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,fp8,15,0.13619200388590494
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,1,0.011535999675591787
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,fp8,63,0.14298666516939798
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,63,0.13686933120091757
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,1,0.012538666526476542
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,fp8,31,0.1418346663316091
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,7,0.011567999919255575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,3,0.011530666301647821
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,127,0.14959466457366943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,15,0.011365332951148352
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,15,0.012837332983811697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,7,0.01210133358836174
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,3,0.012442667037248611
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,fp8,127,0.17470399538675943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,31,0.0124746672809124
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,63,0.011760000139474869
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,63,0.01250133290886879
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,127,0.011349332829316458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,255,0.012698666503032049
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,255,0.014752000570297241
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,511,0.014261333892742792
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,31,0.011535999675591787
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,511,0.015685333559910457
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,127,0.013514666507641474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,1023,0.016197333733240765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,255,0.22667733828226724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,1023,0.018863999595244724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,2047,0.02067199970285098
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,fp8,255,0.2086133360862732
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,2047,0.022991999983787537
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,16383,0.08364799618721008
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,4095,0.029029332101345062
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,4095,0.03270933280388514
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,8191,0.04773333172003428
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,16383,0.09590933720270793
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,1,0.011802667131026586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,32767,0.1843199928601583
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,1,0.01249066616098086
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,3,0.011498666057984034
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,3,0.012698666503032049
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,7,0.011685332904259363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,8191,0.05299733579158783
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,7,0.012794667234023413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,15,0.011674666156371435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,15,0.012383999923865
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,32767,0.15230933825174967
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,31,0.011509332805871964
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,31,0.012725333372751871
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,63,0.011706666400035223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,63,0.012810666114091873
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,127,0.01198400060335795
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,511,0.016800000021855038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,127,0.013797332843144735
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,255,0.012778667112191519
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,255,0.016410666207472484
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,511,0.014741333822409311
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,2047,0.023711999257405598
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,1023,0.01708799973130226
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,1023,0.019141333798567455
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,8191,0.050800000627835594
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,2047,0.021269333859284718
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,4095,0.031152000029881794
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,4095,0.03358400116364161
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,8191,0.05481599768002828
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,16383,0.08411199847857158
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,16383,0.10149866342544556
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,32767,0.15413866440455118
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,32767,0.1839359998703003
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,1,0.012458667159080505
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,1,0.013658666362365087
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,3,0.012293333808581034
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,3,0.012885333349307379
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,7,0.012560000022252401
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,7,0.012874666601419449
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,15,0.012383999923865
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,15,0.012842666357755661
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,31,0.012703999876976013
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,31,0.012965332716703415
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,63,0.012639999389648438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,63,0.013189333180586496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,127,0.012517333030700684
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,127,0.014650666465361914
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,255,0.013631999492645264
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,255,0.016117333124081295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,511,0.015317333241303762
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,511,0.01639466608564059
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,1023,0.017157333592573803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,4095,0.03828266759713491
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,1023,0.018906666586796444
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,2047,0.02183466653029124
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,2047,0.023552000522613525
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,4095,0.033941333492596946
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,16383,0.09463466207186381
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,8191,0.06217066446940104
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,8191,0.056330665946006775
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,16383,0.12818666299184164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,32767,0.1942453384399414
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,32767,0.1730133295059204
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,1,0.012175999581813812
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,1,0.013280000537633896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,3,0.012063999970753988
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,3,0.013621332744757334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,7,0.012421333541472753
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,7,0.01341333364446958
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,15,0.013194666554530462
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,15,0.012170666207869848
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,31,0.012367999802033106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,31,0.013264000415802002
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,255,0.01394133393963178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,63,0.012175999581813812
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,63,0.013957332819700241
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,127,0.012725333372751871
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,127,0.014890667051076889
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,255,0.016437333077192307
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,511,0.015541333705186844
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,511,0.01708799973130226
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,4095,0.05894933144251505
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,1023,0.017616000026464462
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,1023,0.019130667050679524
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,2047,0.036917333801587425
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,2047,0.02831999957561493
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,4095,0.04266133407751719
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,8191,0.10201066732406616
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,8191,0.06676800052324931
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,16383,0.19014400243759155
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,16383,0.13125866651535034
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,32767,0.365994652112325
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,32767,0.23489065965016684
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,1,0.039077334105968475
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,fp8,1,0.03956266740957896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,3,0.03899733225504557
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,fp8,3,0.039018665750821434
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,fp8,7,0.03915199885765711
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,15,0.03977599988381068
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,fp8,15,0.03973866750796636
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,7,0.03899200012286504
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,31,0.04020266731580099
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,fp8,31,0.03929600119590759
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,63,0.04665066798528036
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,fp8,63,0.039264000952243805
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,1,0.0650079995393753
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,fp8,127,0.054192001620928444
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,127,0.05020800232887268
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,fp8,1,0.06312533219655354
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,3,0.06474133332570393
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,fp8,3,0.06333333253860474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,7,0.06537599861621857
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,15,0.06750933329264323
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,fp8,7,0.06312533219655354
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,fp8,15,0.06347733239332835
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,31,0.07233066856861115
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,fp8,31,0.06444266438484192
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,63,0.07444266478220622
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,fp8,63,0.0724426656961441
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,127,0.08070933322111766
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,fp8,1,0.1120693286259969
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,1,0.11098666985829671
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,fp8,127,0.08895466725031535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,3,0.11117866635322571
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,fp8,3,0.11291199922561646
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,7,0.11206400394439697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,fp8,7,0.11268267035484314
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,15,0.11876266201337178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,fp8,15,0.11270933349927266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,31,0.12294399738311768
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,fp8,31,0.11972266435623169
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,63,0.12558933099110922
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,fp8,63,0.12378666798273723
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,1,0.21157866716384888
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,127,0.13871467113494873
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,fp8,127,0.1545973320802053
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,fp8,1,0.20582934220631918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,3,0.21960532665252686
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,fp8,3,0.20601065953572592
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,7,0.2286506692568461
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,fp8,7,0.21002666155497232
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,15,0.22949333985646567
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,fp8,15,0.22138667106628418
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,fp8,31,0.2220426599184672
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,31,0.23221866289774576
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,63,0.23540266354878744
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,fp8,63,0.22459199031194052
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,fp8,127,0.29257599512736004
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,127,0.2521973252296448
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,1,0.07353599866231282
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,fp8,1,0.06646933158238728
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,fp8,3,0.06697600086530049
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,3,0.07291199763615926
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,7,0.07451733450094859
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,fp8,7,0.06770133475462596
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,15,0.0765066643555959
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,fp8,15,0.07009600102901459
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,31,0.07666133344173431
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,fp8,31,0.07586133480072021
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,1,0.12755200266838074
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,63,0.07806399961312611
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,fp8,63,0.07898666461308797
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,fp8,1,0.11454932888348897
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,3,0.12866666913032532
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,fp8,3,0.1151146690050761
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,7,0.12985066572825113
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,fp8,7,0.11778133114178975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,15,0.13168533643086752
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,fp8,15,0.1249120036760966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,31,0.13061867157618204
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,fp8,31,0.12731732924779257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,63,0.1330560048421224
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,1,0.21072532733281454
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,fp8,1,0.21183466911315918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,fp8,63,0.12894933422406515
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,3,0.21662400166193643
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,fp8,3,0.21394133567810059
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,7,0.22043200333913168
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,fp8,7,0.21855467557907104
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,15,0.2258560061454773
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,fp8,15,0.22368532419204712
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,31,0.22961066166559854
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,fp8,31,0.22739199797312418
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,63,0.237664004166921
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,fp8,63,0.23206400871276855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,1,0.44250667095184326
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,fp8,1,0.4166933298110962
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,3,0.4448106686274211
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,fp8,3,0.4248693386713664
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,7,0.4443306525548299
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,1,0.01181866725285848
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,1,0.012847999731699625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,3,0.011850666254758835
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,fp8,7,0.4262293179829915
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,15,0.4487573305765788
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,3,0.012928000340859095
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,fp8,15,0.4294826587041219
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,7,0.011802667131026586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,15,0.011829332758982977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,7,0.012885333349307379
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,15,0.012986666212479273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,31,0.011877333124478659
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,63,0.011936000237862269
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,31,0.012800000607967377
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,63,0.012965332716703415
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,31,0.4459199905395508
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,127,0.011941333611806234
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,127,0.014159999787807465
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,255,0.012906666845083237
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,fp8,31,0.4286400079727173
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,255,0.014874666929244995
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,511,0.015285332997639975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,1023,0.022106667359670002
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,1023,0.019658666104078293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,511,0.01725333308180173
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,2047,0.028090665737787884
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,4095,0.046112000942230225
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,8191,0.08305599788824718
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,4095,0.051226665576299034
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,2047,0.03163733333349228
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,8191,0.09300266702969869
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,16383,0.15130133430163065
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,16383,0.18429332971572876
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,1,0.012261333564917246
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,1,0.013050666699806849
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,3,0.012106666962305704
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,3,0.013594667116800943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,7,0.01211200033624967
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,7,0.01312000056107839
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,63,0.45324798425038654
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,31,0.011999999483426413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,15,0.012149333953857422
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,15,0.013301332791646322
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,fp8,63,0.4319893519083659
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,31,0.013434667140245438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,127,0.014442666123310724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,63,0.01202133297920227
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,63,0.013584000368913015
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,255,0.013679999858140945
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,255,0.015647999942302704
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,511,0.017802666872739792
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,1023,0.019776000330845516
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,1023,0.022917332748572033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,127,0.012080000092585882
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,2047,0.028927999238173168
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,2047,0.032831999162832894
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,4095,0.04939199984073639
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,4095,0.05422399938106537
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,511,0.015397333850463232
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,8191,0.08482666810353597
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,8191,0.10015466809272766
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,16383,0.15437333782513937
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,1,0.012213333199421564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,16383,0.18403732776641846
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,7,0.01257066677014033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,1,0.013631999492645264
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,3,0.01231466606259346
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,3,0.013386666774749756
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,7,0.013290667285521826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,31,0.01368533323208491
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,15,0.012421333541472753
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,15,0.013594667116800943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,31,0.012762666990359625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,63,0.012714666624863943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,63,0.013376000026861826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,127,0.012863999853531519
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,127,0.014970666418472925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,255,0.014352000008026758
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,255,0.015882667154073715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,511,0.016410666207472484
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,511,0.018218666315078735
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,1023,0.021482666333516438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,1023,0.02279466638962428
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,2047,0.03499199946721395
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,2047,0.03678400069475174
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,4095,0.05946666498978933
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,4095,0.05599466462930044
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,8191,0.10520000259081523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,8191,0.09514133135477702
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,16383,0.19352000951766968
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,1,0.0124746672809124
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,16383,0.17281599839528403
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,1,0.013861333330472311
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,3,0.012245333443085352
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,3,0.013882666826248169
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,7,0.01258133351802826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,7,0.013642666240533194
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,15,0.01250133290886879
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,15,0.013818666338920593
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,31,0.012442667037248611
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,31,0.013877333452304205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,63,0.012453333785136541
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,63,0.013823999712864557
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,127,0.013013333082199097
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,127,0.015376000354687372
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,255,0.014202666779359182
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,255,0.016058667252461117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,511,0.01646399994691213
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,511,0.018746666610240936
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,1023,0.03524799893299738
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,1023,0.024160000185171764
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,2047,0.05681066711743673
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,2047,0.040778666734695435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,4095,0.10113599896430969
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,4095,0.06480533381303151
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,8191,0.1879733403523763
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,fp8,1,0.013157332936922709
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,8191,0.11380799611409505
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,16383,0.36376531918843585
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,1,0.012522666404644648
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,16383,0.20789867639541626
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,3,0.012053333222866058
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,fp8,3,0.013573333621025085
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,7,0.012256000190973282
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,fp8,7,0.01313599944114685
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,15,0.01231466606259346
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,fp8,15,0.013125333935022354
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,31,0.01251199965675672
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,fp8,31,0.01339200014869372
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,63,0.012154666086037954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,fp8,63,0.01379199946920077
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,127,0.012080000092585882
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,fp8,127,0.015013333410024643
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,255,0.013541333377361298
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,fp8,255,0.015882667154073715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,511,0.01573333392540614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,fp8,511,0.017968000223239262
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,1023,0.019866666446129482
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,fp8,1023,0.022511998812357586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,2047,0.029898665845394135
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,fp8,2047,0.03276266654332479
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,4095,0.0488373339176178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,fp8,4095,0.05343999962011973
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,8191,0.08379733562469482
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,fp8,8191,0.10025599598884583
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,1,0.012074666718641916
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,fp8,1,0.013365333278973898
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,3,0.012533333152532578
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,fp8,3,0.013904000322024027
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,7,0.012421333541472753
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,fp8,7,0.013418667018413544
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,15,0.012346666306257248
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,fp8,15,0.013199999928474426
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,31,0.01227733368674914
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,fp8,31,0.013418667018413544
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,63,0.01258133351802826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,255,0.013839999834696451
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,fp8,63,0.013616000612576803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,127,0.012367999802033106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,fp8,127,0.014869333555301031
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,fp8,255,0.015274666249752045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,511,0.01540800059835116
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,fp8,511,0.01854933301607768
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,1023,0.02062400057911873
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,fp8,1023,0.022656001150608063
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,2047,0.036618667344252266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,fp8,2047,0.033914667864640556
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,4095,0.061493332187334694
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,fp8,8191,0.10001599788665771
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,fp8,4095,0.057248001297314964
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,8191,0.11116266250610352
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,1,0.012485332787036896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,fp8,1,0.01470400020480156
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,3,0.012634667257467905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,7,0.012469333906968435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,fp8,3,0.013669333110253016
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,fp8,7,0.01357866699496905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,15,0.012597333639860153
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,fp8,15,0.014074667046467463
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,31,0.012736000120639801
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,fp8,31,0.014090667168299357
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,63,0.012768000364303589
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,fp8,63,0.01370666672786077
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,127,0.0129120002190272
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,fp8,127,0.015130666395028433
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,255,0.01441066712141037
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,fp8,255,0.016208000481128693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,511,0.017551999539136887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,fp8,511,0.018853332847356796
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,1023,0.03630933413902918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,fp8,1023,0.02386133372783661
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,2047,0.05713599920272827
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,fp8,2047,0.04083733260631561
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,4095,0.10139200091362
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,fp8,4095,0.06550399959087372
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,8191,0.18939733505249023
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,3,0.016303999970356624
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,fp8,8191,0.1149120032787323
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,1,0.016458666572968166
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,fp8,1,0.017808000246683758
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,15,0.016629333297411602
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,fp8,15,0.01777600000301997
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,fp8,3,0.0176959993938605
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,7,0.016549333930015564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,fp8,7,0.01817600056529045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,31,0.016783999900023144
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,fp8,31,0.017893332988023758
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,63,0.01657066618402799
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,fp8,63,0.018618666877349217
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,127,0.016832000265518825
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,fp8,127,0.020682666450738907
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,255,0.02045866722861926
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,fp8,255,0.022405333817005157
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,511,0.038106667498747505
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,fp8,511,0.028944000601768494
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,1023,0.059621334075927734
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,fp8,1023,0.04451199869314829
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,2047,0.101583997408549
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,fp8,2047,0.06900266806284587
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,4095,0.18980799118677774
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,fp8,4095,0.11618133385976155
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,1,0.011450666934251785
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,1,0.012906666845083237
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,8191,0.36208534240722656
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,fp8,8191,0.21173334121704102
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,3,0.011338666081428528
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,3,0.012426666915416718
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,7,0.011519999553759893
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,7,0.012725333372751871
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,15,0.011413333316644033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,15,0.01268799975514412
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,31,0.011488000551859537
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,31,0.012602667013804117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,63,0.011402666568756104
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,63,0.012821332861979803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,127,0.011493333925803503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,127,0.013557333499193192
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,255,0.012597333639860153
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,255,0.013839999834696451
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,511,0.012565333396196365
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,511,0.013642666240533194
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,1023,0.013114667187134424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,1023,0.014746667196353277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,2047,0.014127999544143677
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,2047,0.015834666788578033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,4095,0.015658666690190632
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,4095,0.01687466725707054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,8191,0.017717332889636356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,8191,0.019472000499566395
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,16383,0.02260799954334895
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,16383,0.02483733246723811
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,32767,0.03250666707754135
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,32767,0.0349386657277743
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,1,0.011418666690587997
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,1,0.012240000069141388
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,3,0.01139733319481214
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,3,0.012794667234023413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,7,0.011365332951148352
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,7,0.012560000022252401
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,15,0.011589333415031433
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,15,0.012853333105643591
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,31,0.011173332730929056
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,31,0.012837332983811697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,63,0.011407999942700068
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,63,0.012458667159080505
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,127,0.011567999919255575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,127,0.013669333110253016
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,255,0.012517333030700684
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,255,0.01333333303531011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,511,0.012874666601419449
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,511,0.014479999740918478
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,1023,0.013258667041858038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,4095,0.0174346665541331
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,1023,0.014906667172908783
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,2047,0.014405333747466406
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,2047,0.01714133347074191
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,4095,0.016271999726692837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,8191,0.019354666272799175
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,8191,0.020234666764736176
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,16383,0.023210667073726654
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,16383,0.02554133286078771
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,32767,0.03358400116364161
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,32767,0.03551466763019562
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,1,0.011690666278203329
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,1,0.01232533281048139
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,3,0.011813333878914515
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,3,0.01259200026591619
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,7,0.011711999773979187
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,7,0.012821332861979803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,15,0.011695999652147293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,15,0.012949333836634954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,31,0.011648000528415045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,31,0.01232533281048139
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,63,0.011861333002646765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,63,0.012533333152532578
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,127,0.01192533348997434
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,127,0.013744000345468521
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,255,0.01259200026591619
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,255,0.014005333185195923
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,511,0.013130666067202887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,511,0.014783999572197596
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,1023,0.014495999862750372
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,1023,0.015423999478419622
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,2047,0.01533866673707962
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,2047,0.017360000560681026
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,4095,0.016650666793187458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,4095,0.01786133274435997
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,8191,0.018800000349680584
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,8191,0.020256000260512035
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,16383,0.02679466704527537
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,3,0.011706666400035223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,16383,0.025797332326571148
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,32767,0.04079466561476389
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,32767,0.037178667883078255
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,1,0.011727999895811081
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,1,0.012655999511480331
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,3,0.012362666428089142
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,7,0.011733333269755045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,7,0.012885333349307379
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,15,0.011792000383138657
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,15,0.012858666479587555
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,31,0.011733333269755045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,31,0.012784000486135483
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,63,0.011898666620254517
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,63,0.012506666282812754
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,127,0.011786667009194693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,127,0.013946666071812311
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,255,0.012901333471139273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,255,0.01463466634353002
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,511,0.013728000223636627
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,511,0.015381333728631338
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,1023,0.014709333578745524
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,1023,0.015594666202863058
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,2047,0.015893333901961643
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,2047,0.01672533278663953
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,4095,0.017157333592573803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,4095,0.018144000321626663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,8191,0.019845332950353622
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,8191,0.020853333175182343
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,1,0.011418666690587997
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,16383,0.037632000943024956
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,16383,0.02898666759332021
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,32767,0.05956799785296122
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,32767,0.04791999856630961
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,1,0.01250133290886879
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,3,0.011509332805871964
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,3,0.012703999876976013
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,7,0.011328000575304031
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,7,0.0129120002190272
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,15,0.01126933346192042
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,15,0.012901333471139273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,31,0.011370666325092316
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,31,0.013061333447694778
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,63,0.011429333438475927
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,63,0.012293333808581034
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,127,0.011706666400035223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,127,0.013349333157142004
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,255,0.012725333372751871
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,255,0.013610667238632837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,511,0.012949333836634954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,511,0.014261333892742792
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,1023,0.013722666849692663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,1023,0.015626666446526844
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,2047,0.015040000279744467
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,2047,0.01642666632930438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,4095,0.01756799966096878
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,4095,0.01918399954835574
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,8191,0.021359999974568684
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,8191,0.023311999936898548
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,16383,0.02995733420054118
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,16383,0.033189333975315094
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,32767,0.048298666874567665
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,32767,0.052570665876070656
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,1,0.011658667276302973
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,1,0.012810666114091873
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,3,0.011450666934251785
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,3,0.012762666990359625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,7,0.0116799995303154
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,7,0.012341332932313284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,15,0.011711999773979187
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,15,0.012714666624863943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,31,0.011674666156371435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,31,0.013050666699806849
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,63,0.011541333049535751
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,63,0.012869333227475485
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,127,0.011674666156371435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,127,0.013552000125249227
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,255,0.012815999488035837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,255,0.013898666948080063
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,511,0.01321600005030632
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,511,0.015061333775520325
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,1023,0.014261333892742792
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,4095,0.019765333582957584
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,1023,0.016154666741689045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,2047,0.016234666109085083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,2047,0.017082666357358296
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,4095,0.018383999665578205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,8191,0.021727999051411945
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,8191,0.024133334557215374
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,16383,0.030165334542592365
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,16383,0.034458667039871216
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,32767,0.04986133178075155
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,32767,0.055642664432525635
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,1,0.011701333026091257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,1,0.012858666479587555
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,3,0.011733333269755045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,3,0.012655999511480331
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,7,0.011546666423479715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,7,0.01257066677014033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,15,0.011802667131026586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,15,0.013232000172138214
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,31,0.011711999773979187
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,31,0.012917333592971167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,255,0.013034666577974955
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,63,0.011733333269755045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,63,0.012650666137536367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,127,0.011722666521867117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,127,0.013754667093356451
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,255,0.014858666807413101
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,511,0.013733333597580591
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,511,0.015216000378131866
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,1023,0.01534933348496755
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,1023,0.016122666498025257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,2047,0.01672533278663953
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,2047,0.01725333308180173
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,4095,0.01846933364868164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,4095,0.019999999552965164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,8191,0.025802666942278545
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,8191,0.0242399995525678
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,16383,0.0425600012143453
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,16383,0.03691199918588003
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,32767,0.06283199787139893
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,32767,0.058687999844551086
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,1,0.01190399999419848
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,1,0.012928000340859095
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,3,0.012106666962305704
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,3,0.0129120002190272
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,7,0.011893333246310553
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,7,0.01303999995191892
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,15,0.011786667009194693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,15,0.012741333494583765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,31,0.012085333466529846
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,31,0.013077333569526672
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,63,0.011877333124478659
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,255,0.013365333278973898
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,63,0.013242666920026144
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,127,0.012266666938861212
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,127,0.014597332725922266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,255,0.014837333311637243
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,511,0.013823999712864557
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,511,0.01534933348496755
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,1023,0.015365333606799444
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,1023,0.016399999459584553
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,2047,0.017269333203633625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,2047,0.01738133281469345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,4095,0.019205333044131596
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,4095,0.020293333878119785
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,8191,0.037317333122094475
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,8191,0.025290665527184803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,16383,0.058975999553998314
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,16383,0.047509332497914634
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,32767,0.10241066416104634
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,32767,0.07554133236408234
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,1,0.013232000172138214
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,fp8,1,0.014090667168299357
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,3,0.013114667187134424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,fp8,3,0.013701333353916803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,7,0.013232000172138214
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,fp8,7,0.013541333377361298
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,15,0.01301866645614306
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,fp8,15,0.013967999567588171
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,31,0.013178666432698568
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,fp8,31,0.013477332890033722
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,63,0.013343999783198038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,fp8,63,0.013904000322024027
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,127,0.01313599944114685
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,fp8,127,0.014938666174809137
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,255,0.014885333677132925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,fp8,255,0.016271999726692837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,511,0.016282666474580765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,fp8,511,0.01878400022784869
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,1023,0.02178666740655899
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,fp8,1023,0.022826666633288067
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,2047,0.038506666819254555
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,fp8,2047,0.03523733218510946
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,4095,0.06092800199985504
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,fp8,1,0.014655999839305878
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,1,0.013408000270525614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,fp8,4095,0.05845333139101664
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,3,0.01331199953953425
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,fp8,3,0.014149333039919535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,7,0.013093333691358566
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,fp8,7,0.014463999619086584
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,fp8,15,0.014544000228246054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,15,0.013327999661366144
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,31,0.01332266628742218
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,63,0.013221333424250284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,fp8,31,0.014362666755914688
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,255,0.015477333217859268
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,fp8,63,0.014101333916187286
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,127,0.013493333011865616
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,fp8,127,0.015775999675194424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,fp8,255,0.01629866659641266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,1023,0.03590933233499527
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,511,0.017738666385412216
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,fp8,511,0.019343999524911244
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,fp8,1023,0.026133333643277485
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,2047,0.05840533475081126
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,4095,0.10073066751162212
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,fp8,2047,0.04179200033346812
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,3,0.017312000195185345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,fp8,4095,0.06666666766007741
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,1,0.017045332739750545
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,fp8,1,0.018453333526849747
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,fp8,3,0.0183146670460701
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,fp8,7,0.018288000176350277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,7,0.016986666868130367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,fp8,15,0.018165333817402523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,15,0.017312000195185345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,fp8,63,0.018426666657129925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,31,0.017173333714405697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,fp8,31,0.018063999712467194
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,63,0.017498667041460674
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,127,0.017653333644072216
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,fp8,127,0.021002667645613354
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,255,0.021477334201335907
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,fp8,255,0.022522665560245514
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,511,0.039093332986036934
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,fp8,511,0.02923733244339625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,1023,0.06089066465695699
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,fp8,1023,0.04586133360862732
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,2047,0.10302399595578511
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,fp8,2047,0.07061333457628886
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,fp8,4095,0.11897066235542297
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,4095,0.19010667006174722
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,1,0.024085332949956257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,fp8,1,0.02535466601451238
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,3,0.02386666586001714
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,fp8,3,0.025594666600227356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,7,0.02385599911212921
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,fp8,15,0.025583999852339428
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,fp8,7,0.02571200082699458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,31,0.0239680012067159
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,15,0.02422933280467987
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,fp8,31,0.025989333788553875
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,63,0.02439466615517934
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,fp8,63,0.025333332518736523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,127,0.024864000578721363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,fp8,127,0.030554667115211487
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,255,0.043231998880704246
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,fp8,255,0.036346666514873505
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,511,0.06280000011126201
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,fp8,511,0.0516480008761088
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,1023,0.10492799679438274
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,fp8,1023,0.07576000193754832
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,2047,0.1912213365236918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,fp8,2047,0.12344533205032349
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,1,0.011877333124478659
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,3,0.01091733326514562
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,4095,0.3630613485972087
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,1,0.011215999722480774
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,7,0.011786667009194693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,3,0.012266666938861212
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,7,0.01089599976936976
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,fp8,4095,0.2169813315073649
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,15,0.010837333897749582
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,15,0.012522666404644648
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,31,0.011402666568756104
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,31,0.011839999506870905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,63,0.011221333096424738
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,63,0.012175999581813812
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,127,0.011482667177915573
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,127,0.013610667238632837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,255,0.0124746672809124
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,255,0.013776000589132309
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,511,0.013461332768201828
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,511,0.014906667172908783
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,1023,0.014922666052977243
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,1023,0.01605333387851715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,2047,0.016154666741689045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,2047,0.018474667022625606
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,4095,0.020394666741291683
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,4095,0.023007998863856
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,8191,0.028890666862328846
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,8191,0.03299200038115183
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,16383,0.04717866579691569
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,16383,0.052655999859174095
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,32767,0.08380800485610962
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,32767,0.09755733609199524
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,1,0.011450666934251785
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,1,0.01209066684047381
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,3,0.011183999478816986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,3,0.012335999558369318
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,7,0.01110400011142095
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,7,0.012869333227475485
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,15,0.01119999960064888
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,15,0.012432000289360682
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,31,0.011424000064531961
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,31,0.012117333710193634
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,63,0.011130666981140772
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,63,0.012346666306257248
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,127,0.011114666859308878
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,127,0.013957332819700241
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,255,0.012938667088747025
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,255,0.014538666854302088
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,511,0.014197333405415217
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,511,0.01570133368174235
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,1023,0.015498666713635126
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,1023,0.016458666572968166
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,2047,0.016458666572968166
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,8191,0.03349333256483078
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,16383,0.050026665131251015
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,2047,0.019461333751678467
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,4095,0.021114667256673176
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,4095,0.0242399995525678
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,8191,0.030271999537944794
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,16383,0.05423999826113383
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,32767,0.08457600076993306
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,32767,0.10057066877683003
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,1,0.011434666812419891
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,1,0.013061333447694778
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,3,0.011594666788975397
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,3,0.012768000364303589
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,7,0.011642667154471079
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,7,0.01302933320403099
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,15,0.011472000430027643
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,15,0.012469333906968435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,31,0.011509332805871964
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,31,0.013034666577974955
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,63,0.01156266654531161
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,63,0.012789333860079447
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,127,0.011882666498422623
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,127,0.014677333335081736
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,255,0.013104000439246496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,255,0.014485333114862442
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,511,0.01526933287580808
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,511,0.01580799991885821
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,1023,0.016480000068744022
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,1023,0.016800000021855038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,2047,0.016901332885026932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,2047,0.019493332753578823
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,4095,0.02165333429972331
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,4095,0.02425066630045573
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,8191,0.04232533276081085
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,8191,0.03665599972009659
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,16383,0.06075733403364817
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,16383,0.058592001597086586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,32767,0.10587199529012044
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,32767,0.10096533099810283
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,1,0.01232533281048139
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,1,0.013141332815090815
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,3,0.012015999605258306
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,3,0.013584000368913015
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,7,0.012058666596810022
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,7,0.013487999637921652
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,15,0.0120319997270902
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,15,0.013450667262077332
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,31,0.012175999581813812
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,31,0.01357866699496905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,63,0.012383999923865
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,63,0.013264000415802002
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,127,0.012128000458081564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,127,0.014368000129858652
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,255,0.013839999834696451
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,255,0.014794666320085526
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,511,0.014912000546852747
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,511,0.016410666207472484
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,1023,0.016410666207472484
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,1023,0.016949333250522614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,2047,0.017925333231687546
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,2047,0.019093333433071773
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,4095,0.0374293327331543
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,4095,0.024800000091393787
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,8191,0.05853866537412008
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,8191,0.04286933441956838
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,16383,0.1029813289642334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,16383,0.06704000135262807
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,32767,0.1896479924519857
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,32767,0.11467732985814412
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,1,0.01452800010641416
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,fp8,1,0.015466666469971338
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,3,0.014389333625634512
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,fp8,3,0.015802666544914246
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,7,0.014416000495354334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,fp8,7,0.015637333194414776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,15,0.014389333625634512
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,fp8,15,0.015418666104475657
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,31,0.014544000228246054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,fp8,31,0.015599999576807022
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,63,0.014495999862750372
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,fp8,63,0.015791999797026317
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,127,0.014864000181357065
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,fp8,127,0.017050666113694508
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,255,0.016800000021855038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,fp8,255,0.017456000049908955
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,511,0.018933333456516266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,fp8,511,0.020314666132132213
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,1023,0.037802666425704956
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,fp8,1023,0.026394667724768322
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,1,0.017781333376963932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,fp8,1,0.019050666441520054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,2047,0.05927466849486033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,3,0.0179626668492953
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,fp8,2047,0.04388799766699473
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,fp8,3,0.019066666563351948
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,7,0.017887999614079792
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,fp8,15,0.018976000448067982
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,15,0.01798933371901512
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,fp8,7,0.019317333896954853
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,31,0.018229333062966663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,fp8,31,0.019567999988794327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,fp8,63,0.019296000401178997
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,127,0.018565333137909572
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,fp8,127,0.021903999149799347
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,63,0.018058666338523228
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,255,0.021877333521842957
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,fp8,255,0.023386667172114056
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,511,0.03988266736268997
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,fp8,511,0.03265066693226496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,1023,0.062362665931383766
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,fp8,1023,0.04765866696834564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,1,0.0244159996509552
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,fp8,1,0.026591998835404713
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,3,0.024522667129834492
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,fp8,3,0.02677333354949951
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,2047,0.10512533783912659
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,fp8,2047,0.07302933434645335
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,15,0.024469333390394848
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,7,0.024485332270463307
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,fp8,15,0.026538667579491932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,fp8,7,0.02649066597223282
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,31,0.024661332368850708
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,fp8,31,0.02677333354949951
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,fp8,63,0.026367999613285065
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,fp8,127,0.031152000029881794
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,127,0.02718399961789449
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,255,0.04305600126584371
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,63,0.024538666009902954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,fp8,255,0.035232000052928925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,fp8,511,0.05333866675694784
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,511,0.06599999964237213
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,2047,0.192303995291392
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,1,0.03752533346414566
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,1023,0.10688533385594685
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,fp8,2047,0.12571199735005698
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,fp8,1023,0.0766186664501826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,fp8,3,0.04058666775623957
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,fp8,1,0.04001600046952566
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,7,0.03754133234421412
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,3,0.03749866783618927
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,15,0.037471999724706016
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,fp8,7,0.04072533299525579
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,31,0.037733333806196846
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,fp8,31,0.04025600105524063
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,fp8,15,0.04035733391841253
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,63,0.039290666580200195
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,fp8,63,0.041034666200478874
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,127,0.04660800099372864
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,fp8,127,0.05226666728655497
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,255,0.06964799761772156
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,1023,0.1950613260269165
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,fp8,255,0.06320533156394958
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,fp8,1023,0.1328373352686564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,2047,0.36690668265024823
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,fp8,511,0.08747200171152751
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,511,0.11040533582369487
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,fp8,2047,0.23094934225082397
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,1,0.020869334538777668
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,fp8,1,0.022240000466505688
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,3,0.02072000006834666
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,fp8,3,0.022597332795461018
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,15,0.02081599955757459
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,fp8,15,0.022869333624839783
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,31,0.021007999777793884
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,7,0.0206986665725708
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,fp8,7,0.022304000953833263
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,fp8,31,0.022543999056021374
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,63,0.02067199970285098
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,fp8,63,0.02298133323589961
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,fp8,127,0.025194667279720306
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,127,0.021365332106749218
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,255,0.026538667579491932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,fp8,255,0.026869334280490875
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,511,0.04400533437728882
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,1,0.02683199942111969
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,fp8,511,0.034703999757766724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,fp8,1,0.029338667790095013
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,3,0.02697066714366277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,fp8,3,0.02942399928967158
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,7,0.027093333502610523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,fp8,7,0.029530666768550873
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,15,0.02696000039577484
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,fp8,15,0.02882133424282074
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,fp8,31,0.029189333319664
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,31,0.02699200063943863
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,63,0.0272533322374026
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,fp8,63,0.02942399928967158
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,127,0.02941333254178365
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,fp8,127,0.03369600077470144
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,255,0.047237331668535866
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,fp8,255,0.04046933352947235
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,1,0.03996799886226654
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,511,0.07063466807206471
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,fp8,511,0.05651199817657471
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,fp8,1,0.04321066538492838
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,3,0.03994133323431015
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,fp8,3,0.04307733476161957
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,7,0.03996799886226654
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,fp8,7,0.04277333120505015
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,15,0.040175999204317726
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,fp8,15,0.04364799956480662
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,fp8,31,0.0432640016078949
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,63,0.04298666616280874
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,fp8,63,0.04321066538492838
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,31,0.039850667119026184
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,127,0.05131733417510986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,fp8,127,0.05473066866397858
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,255,0.07123200098673503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,fp8,255,0.06764799853165944
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,1,0.06574399769306183
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,fp8,1,0.06975999971230824
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,fp8,511,0.09212266405423482
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,3,0.06549866497516632
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,511,0.11479999621709187
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,fp8,3,0.06969599922498067
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,7,0.06584533552328746
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,fp8,7,0.07062399884064992
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,15,0.06592000027497609
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,fp8,15,0.07016000151634216
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,fp8,31,0.07023466626803081
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,fp8,63,0.07311466832955678
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,31,0.06890133519967397
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,127,0.07897066573301952
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,fp8,127,0.09292266766230266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,63,0.0728000005086263
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,255,0.12212799986203511
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,fp8,255,0.10771200060844421
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,1,0.037808001041412354
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,fp8,1,0.0391146664818128
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,511,0.20488532384236655
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,3,0.037685332198937736
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,fp8,511,0.16220800081888834
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,fp8,3,0.039077334105968475
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,7,0.037834666669368744
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,fp8,7,0.03924266745646795
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,15,0.03756266583998998
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,31,0.03779733429352442
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,63,0.03791466603676478
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,fp8,15,0.039034667114416756
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,fp8,63,0.03908266623814901
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,127,0.04278400043646494
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,fp8,31,0.03940266619126002
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,fp8,127,0.04344533383846283
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,255,0.05898133416970571
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,fp8,255,0.0525439977645874
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,1,0.04896533489227295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,fp8,1,0.051674668987592064
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,3,0.0488373339176178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,fp8,3,0.0521919975678126
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,7,0.049082666635513306
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,fp8,7,0.05202133456865946
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,15,0.04952533543109894
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,fp8,15,0.052095999320348106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,31,0.049786667029062905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,fp8,31,0.05171733101209005
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,63,0.052853330969810486
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,fp8,63,0.05236266553401947
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,127,0.06306666632493337
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,fp8,127,0.06408533453941345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,fp8,255,0.0782293329636256
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,255,0.08301333089669545
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,1,0.07529066503047943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,fp8,1,0.0804799993832906
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,3,0.07553066809972127
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,fp8,3,0.081386665503184
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,7,0.0754613329966863
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,fp8,7,0.08084799846013387
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,15,0.07642666498819987
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,31,0.0804799993832906
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,fp8,15,0.08119999865690868
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,fp8,31,0.08070933322111766
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,63,0.08348799745241801
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,fp8,63,0.08563733100891113
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,127,0.09121599793434143
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,fp8,127,0.10338133573532104
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,1,0.12716799974441528
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,fp8,1,0.13340266545613608
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,255,0.13059199849764505
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,fp8,255,0.12169599533081055
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,3,0.12723199526468912
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,fp8,3,0.13524799545605978
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,7,0.12783466776212057
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,fp8,7,0.13380266229311624
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,fp8,15,0.1341600020726522
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,15,0.1330933372179667
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,63,0.13622400164604187
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,31,0.13659200072288513
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,fp8,63,0.1423413356145223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,fp8,31,0.13940800229708353
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,127,0.14842666188875833
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,fp8,127,0.17305066188176474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,1,0.012373333175977072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,3,0.012410666793584824
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,15,0.011589333415031433
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,1,0.011450666934251785
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,3,0.01219733307758967
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,31,0.011349332829316458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,15,0.01251199965675672
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,31,0.012383999923865
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,63,0.011493333925803503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,63,0.012437333663304647
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,127,0.011424000064531961
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,127,0.013408000270525614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,255,0.01257066677014033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,7,0.012495999534924826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,7,0.011530666301647821
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,255,0.014629332969586054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,511,0.014111999422311783
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,511,0.01562133307258288
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,1023,0.016106666376193363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,1023,0.018330667167901993
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,4095,0.032730666299661
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,2047,0.020586666961510975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,2047,0.023056000471115112
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,4095,0.028965334097544353
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,255,0.22129599253336588
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,fp8,255,0.20316267013549805
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,8191,0.04728533327579498
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,32767,0.1842986742655436
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,8191,0.052202666799227394
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,16383,0.08326933284600575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,16383,0.09355733791987102
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,32767,0.154341330130895
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,1,0.012698666503032049
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,3,0.01156266654531161
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,3,0.013776000589132309
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,7,0.011674666156371435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,7,0.012768000364303589
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,1,0.01145600030819575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,15,0.011765333513418833
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,15,0.012576000144084295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,31,0.011535999675591787
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,31,0.012741333494583765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,63,0.011717333147923151
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,63,0.013002666334311167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,127,0.011493333925803503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,127,0.013807999591032663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,255,0.012981332838535309
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,255,0.016255999604860943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,511,0.015322666615247726
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,2047,0.021130666136741638
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,511,0.016688000410795212
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,1023,0.017258666455745697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,1023,0.018981333822011948
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,2047,0.023567999402681988
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,4095,0.0303413321574529
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,16383,0.08653866251309712
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,4095,0.03365333378314972
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,8191,0.05013333261013031
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,8191,0.05384000142415365
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,16383,0.10065066814422607
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,32767,0.15946666399637857
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,1,0.012362666428089142
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,32767,0.1834719975789388
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,1,0.013173333058754602
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,3,0.012080000092585882
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,3,0.013232000172138214
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,7,0.01210133358836174
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,7,0.013487999637921652
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,15,0.012383999923865
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,15,0.01331199953953425
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,31,0.012005332857370377
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,31,0.013338666409254074
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,63,0.012186666329701742
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,63,0.013376000026861826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,127,0.0122079998254776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,127,0.014949332922697067
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,255,0.01434133326013883
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,255,0.015504000087579092
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,511,0.015103999525308609
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,511,0.01692266638080279
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,1023,0.017429333180189133
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,1023,0.01921066641807556
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,2047,0.02205866575241089
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,2047,0.023738667368888855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,16383,0.10713066657384236
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,4095,0.038831998904546104
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,4095,0.03489600121974945
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,8191,0.06202666461467743
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,8191,0.05861866474151611
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,16383,0.10114666819572449
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,32767,0.19619200627009073
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,32767,0.18490666151046753
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,1,0.012442667037248611
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,1,0.013850666582584381
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,3,0.011999999483426413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,3,0.01332266628742218
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,7,0.012122667084137598
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,7,0.013514666507641474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,15,0.012175999581813812
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,15,0.013034666577974955
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,31,0.01239466667175293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,31,0.014042666802803675
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,63,0.012202666451533636
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,63,0.01469333345691363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,127,0.01258133351802826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,127,0.014794666320085526
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,255,0.014090667168299357
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,1023,0.019509332875410717
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,255,0.01632000009218852
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,511,0.015642666568358738
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,2047,0.02629333237806956
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,511,0.017258666455745697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,1023,0.018538666268189747
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,2047,0.03706666578849157
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,4095,0.04270400106906891
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,4095,0.058490668733914696
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,8191,0.10254933436711629
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,8191,0.06660800178845723
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,16383,0.188917338848114
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,16383,0.11407466729482015
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,32767,0.36638398965199787
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,32767,0.20982933044433594
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,fp8,1,0.03878933439652125
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,1,0.038704000413417816
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,3,0.03904533386230469
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,fp8,3,0.039520000418027244
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,7,0.03860266755024592
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,fp8,7,0.039488000174363456
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,fp8,15,0.03912533322970072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,15,0.03869866579771042
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,31,0.038592000802357994
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,63,0.045365333557128906
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,fp8,31,0.039221333960692085
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,fp8,63,0.03938666731119156
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,127,0.0496373325586319
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,fp8,127,0.05314133564631144
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,1,0.06443733473618825
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,fp8,1,0.06298133234182994
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,3,0.06479999919732411
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,fp8,3,0.06298133234182994
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,7,0.0643146683772405
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,fp8,7,0.06298133234182994
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,fp8,15,0.06282133360703786
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,15,0.06688533226648967
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,31,0.07238399982452393
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,fp8,31,0.06352533400058746
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,63,0.07363200187683105
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,fp8,63,0.0721013347307841
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,127,0.07885866860548656
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,fp8,127,0.0883626639842987
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,1,0.1151626706123352
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,fp8,1,0.1136853297551473
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,3,0.11641599734624226
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,fp8,3,0.11346133550008138
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,7,0.11777066191037495
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,15,0.12657066186269125
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,fp8,7,0.11366933584213257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,fp8,15,0.11401599645614624
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,31,0.1275040010611216
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,fp8,31,0.12268267075220744
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,63,0.12999467055002847
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,fp8,63,0.12570133805274963
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,127,0.14020267128944397
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,fp8,1,0.20627733071645102
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,fp8,127,0.15870933731396994
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,1,0.2116853396097819
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,3,0.21364800135294595
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,fp8,3,0.2060533364613851
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,7,0.23008533318837485
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,fp8,7,0.20774932702382407
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,fp8,15,0.21982399622599283
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,15,0.2307786742846171
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,31,0.22984000047047934
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,fp8,31,0.222378671169281
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,63,0.23196800549825033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,fp8,63,0.2241493264834086
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,127,0.25261332591374713
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,fp8,127,0.291157325108846
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,1,0.06621866424878438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,fp8,1,0.06601066887378693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,3,0.06677866478761037
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,fp8,3,0.06609066824118297
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,7,0.06900266806284587
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,fp8,7,0.06638399759928386
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,15,0.07336533566315968
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,fp8,15,0.06680533289909363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,31,0.07513066629568736
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,fp8,31,0.07169599831104279
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,63,0.07621333499749501
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,fp8,1,0.1130506694316864
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,1,0.11762666702270508
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,fp8,63,0.07768533130486806
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,3,0.12266666690508525
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,fp8,3,0.11294933160146077
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,7,0.12796800335248312
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,fp8,7,0.11459733049074809
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,15,0.12982400258382162
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,fp8,15,0.11869333187739055
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,31,0.12930132945378622
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,63,0.13251200318336487
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,fp8,31,0.12679466605186462
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,fp8,63,0.1283680001894633
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,1,0.2269066572189331
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,fp8,1,0.21504533290863037
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,3,0.23643199602762857
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,fp8,3,0.21523199478785196
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,7,0.2388319969177246
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,fp8,7,0.22554665803909302
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,15,0.23988799254099527
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,fp8,15,0.23165865739186606
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,31,0.2410879929860433
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,fp8,31,0.2336746652921041
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,63,0.24361600478490195
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,fp8,63,0.2347093423207601
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,fp8,1,0.40757866700490314
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,1,0.4398719867070516
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,3,0.4442880153656006
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,fp8,3,0.42022931575775146
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,7,0.4451413154602051
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,fp8,7,0.42812267939249676
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,1,0.011850666254758835
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,1,0.012714666624863943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,3,0.01184533288081487
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,3,0.012741333494583765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,7,0.01192533348997434
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,7,0.012890666723251343
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,15,0.444597323735555
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,15,0.011760000139474869
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,15,0.01303999995191892
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,31,0.011685332904259363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,31,0.013023999830087027
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,fp8,15,0.4273173411687215
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,63,0.011834666132926941
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,63,0.013023999830087027
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,127,0.011744000017642975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,31,0.45073068141937256
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,255,0.013104000439246496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,127,0.014074667046467463
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,255,0.01462399959564209
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,fp8,31,0.4323253234227498
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,511,0.01522133375207583
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,1023,0.01951466624935468
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,1023,0.022064000368118286
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,2047,0.02810666710138321
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,2047,0.0322026660044988
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,4095,0.05101333558559418
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,511,0.01757866640885671
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,8191,0.0828000009059906
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,4095,0.04653333127498627
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,8191,0.09473066528638203
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,16383,0.1518239974975586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,16383,0.183786670366923
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,1,0.012400000045696894
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,1,0.013237333546082178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,3,0.012266666938861212
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,3,0.013424000392357508
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,7,0.012058666596810022
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,7,0.013749333719412485
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,15,0.012133333832025528
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,15,0.013429333766301474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,31,0.011978667229413986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,31,0.013445333888133367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,63,0.45129601160685223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,63,0.012341332932313284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,63,0.013471999516089758
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,127,0.01221866657336553
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,127,0.014645333091417948
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,255,0.01393066719174385
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,1023,0.01979200045267741
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,255,0.015504000087579092
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,fp8,63,0.43063998222351074
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,511,0.015397333850463232
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,1023,0.02271466702222824
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,2047,0.02920000006755193
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,2047,0.032842665910720825
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,4095,0.04962133367856344
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,4095,0.05340266724427541
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,511,0.018079999834299088
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,8191,0.08351999521255493
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,16383,0.1535360018412272
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,1,0.013386666774749756
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,16383,0.18311999241511026
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,1,0.0124746672809124
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,3,0.012240000069141388
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,3,0.013621332744757334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,7,0.012416000167528788
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,7,0.01393066719174385
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,15,0.0122079998254776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,8191,0.09942400455474854
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,15,0.013429333766301474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,31,0.01228800043463707
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,127,0.01470400020480156
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,31,0.013552000125249227
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,63,0.01231466606259346
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,63,0.013967999567588171
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,127,0.01257066677014033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,255,0.013663999736309052
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,255,0.015439999600251516
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,511,0.01599466676513354
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,511,0.018101333330074947
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,1023,0.020954666038354237
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,1023,0.023258666197458904
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,2047,0.037205333511034645
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,2047,0.033600000043710075
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,4095,0.06052800019582113
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,4095,0.05715199808279673
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,8191,0.10410133004188538
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,8191,0.09946133693059285
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,16383,0.19383466243743896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,1,0.012629333883523941
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,16383,0.18200000127156576
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,1,0.013647999614477158
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,3,0.012373333175977072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,3,0.013637332866589228
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,7,0.013669333110253016
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,7,0.013728000223636627
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,15,0.014005333185195923
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,15,0.01240533341964086
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,31,0.012448000411192576
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,31,0.013754667093356451
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,63,0.012522666404644648
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,63,0.013776000589132309
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,127,0.01321600005030632
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,127,0.015237333873907724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,255,0.014730667074521383
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,255,0.01621866722901662
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,511,0.01632533346613248
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,511,0.018687999496857326
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,1023,0.034901333351929985
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,1023,0.024421334266662598
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,2047,0.05717866619427999
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,2047,0.040762667854626976
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,4095,0.10076799988746643
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,4095,0.06526400148868561
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,8191,0.1888586680094401
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,8191,0.13029332955678305
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,16383,0.3667626778284709
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,1,0.011941333611806234
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,16383,0.23278399308522543
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,1,0.013242666920026144
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,3,0.012373333175977072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,7,0.012341332932313284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,3,0.013232000172138214
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,7,0.013738666971524557
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,15,0.012223999947309494
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,15,0.013487999637921652
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,31,0.012506666282812754
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,31,0.013189333180586496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,63,0.012159999459981918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,63,0.01332266628742218
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,127,0.01221866657336553
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,127,0.014677333335081736
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,255,0.013621332744757334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,255,0.01594666639963786
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,511,0.015674666812022526
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,511,0.01759999990463257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,1023,0.02006400004029274
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,1023,0.02239466706911723
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,2047,0.02845333268245061
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,2047,0.03314133236805598
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,4095,0.048298666874567665
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,4095,0.053557331363360085
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,8191,0.0846560001373291
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,8191,0.10068800052007039
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,16383,0.15235199530919394
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,16383,0.1843679944674174
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,3,0.013477332890033722
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,1,0.012613333761692047
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,1,0.013503999759753546
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,3,0.012901333471139273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,7,0.012362666428089142
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,7,0.013434667140245438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,15,0.012736000120639801
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,15,0.013503999759753546
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,31,0.012453333785136541
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,31,0.013701333353916803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,63,0.012805332740147909
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,63,0.013690666606028875
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,127,0.012538666526476542
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,127,0.014778666198253632
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,255,0.01421333352724711
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,255,0.015578666081031164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,511,0.016048000504573185
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,2047,0.04162133236726125
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,511,0.01781333362062772
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,1023,0.021557333568731945
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,1023,0.022965334355831146
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,2047,0.0329066663980484
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,4095,0.059994667768478394
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,16383,0.20122667153676352
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,4095,0.055120001236597695
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,8191,0.10586667060852051
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,1,0.014005333185195923
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,8191,0.09412266810735066
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,16383,0.17438934246699014
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,1,0.012682666381200155
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,7,0.013994666437307993
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,3,0.01268799975514412
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,3,0.013807999591032663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,7,0.012682666381200155
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,15,0.012613333761692047
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,15,0.013679999858140945
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,31,0.012554666648308435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,31,0.014096000542243322
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,63,0.012730666746695837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,63,0.014058666924635569
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,127,0.012757333616415659
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,127,0.015418666104475657
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,255,0.014874666929244995
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,255,0.015989333391189575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,511,0.016965333372354507
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,511,0.01886933296918869
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,1023,0.0366293340921402
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,1023,0.024373332659403484
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,2047,0.057034666339556374
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,2047,0.041434665520985924
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,4095,0.10040000081062317
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,4095,0.06534400085608165
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,8191,0.18788800636927286
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,8191,0.11449600259462993
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,1,0.016399999459584553
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,16383,0.36295998096466064
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,1,0.017845333864291508
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,16383,0.20830933252970377
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,3,0.016597333053747814
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,3,0.01775466650724411
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,7,0.01655999943614006
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,7,0.0183146670460701
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,15,0.01655999943614006
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,15,0.018325333793958027
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,31,0.01655999943614006
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,31,0.018042666216691334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,63,0.016773333152135212
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,63,0.018330667167901993
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,127,0.017093333105246227
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,127,0.02075200031201045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,255,0.020367999871571858
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,255,0.022074667116006214
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,511,0.03686933219432831
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,511,0.02870933214823405
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,1023,0.06015466650327047
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,2047,0.06923733154932658
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,1023,0.04444266855716705
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,2047,0.10247466961542766
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,4095,0.18890666961669922
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,4095,0.11658133069674174
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,8191,0.3633600076039632
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,8191,0.23672000567118326
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,1,0.011285333583752314
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,1,0.012122667084137598
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,3,0.011450666934251785
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,3,0.012831999609867731
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,7,0.01156266654531161
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,16383,0.7119786739349365
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,16383,0.39631466070810956
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,7,0.012319999436537424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,15,0.011391999820868174
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,15,0.012981332838535309
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,31,0.011445333560307821
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,31,0.012890666723251343
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,63,0.011503999431928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,63,0.012789333860079447
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,127,0.011264000087976456
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,127,0.013343999783198038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,255,0.0124746672809124
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,255,0.013738666971524557
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,511,0.012768000364303589
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,511,0.013829333086808523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,1023,0.013242666920026144
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,1023,0.014373333503802618
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,2047,0.01413333291808764
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,2047,0.015685333559910457
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,4095,0.01571200042963028
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,4095,0.016607999801635742
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,8191,0.018079999834299088
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,8191,0.020495999604463577
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,16383,0.022687998910744984
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,16383,0.024911999702453613
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,32767,0.03207999964555105
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,32767,0.034927998979886375
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,65535,0.0513919989267985
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,65535,0.05569600065549215
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,1,0.011530666301647821
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,1,0.012533333152532578
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,3,0.011898666620254517
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,3,0.012357333054145178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,7,0.011605333536863327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,7,0.01240533341964086
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,15,0.011839999506870905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,15,0.012351999680201212
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,31,0.011600000162919363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,31,0.012645332763592402
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,255,0.01321600005030632
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,63,0.0116799995303154
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,63,0.012608000387748083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,127,0.011733333269755045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,127,0.01360000049074491
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,255,0.012426666915416718
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,511,0.012896000097195307
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,511,0.015072000523408255
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,1023,0.013280000537633896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,1023,0.014607999473810196
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,2047,0.014501333236694336
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,2047,0.016058667252461117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,16383,0.023013333479563396
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,4095,0.0164533331990242
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,32767,0.034490667283535004
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,4095,0.01706133286158244
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,8191,0.019343999524911244
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,8191,0.01971199984351794
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,16383,0.02532800038655599
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,32767,0.03519999980926514
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,65535,0.053504000107447304
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,65535,0.05765333275000254
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,1,0.011605333536863327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,1,0.012682666381200155
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,15,0.011482667177915573
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,3,0.011695999652147293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,3,0.012853333105643591
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,7,0.011781333635250727
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,7,0.012448000411192576
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,15,0.012879999975363413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,31,0.011653333902359009
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,31,0.012602667013804117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,63,0.01145600030819575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,255,0.014106666048367819
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,63,0.012938667088747025
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,127,0.011658667276302973
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,127,0.014117332796255747
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,255,0.012784000486135483
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,511,0.012858666479587555
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,511,0.014725333700577417
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,1023,0.013653332988421122
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,1023,0.015471999843915304
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,2047,0.015365333606799444
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,2047,0.017456000049908955
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,4095,0.016986666868130367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,4095,0.017909333109855652
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,8191,0.01863466699918111
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,8191,0.02033599962790807
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,16383,0.025813333690166473
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,16383,0.025807999074459076
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,32767,0.039173332353432976
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,32767,0.03718933214743932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,3,0.011727999895811081
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,65535,0.06076799829800924
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,3,0.013189333180586496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,7,0.011813333878914515
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,65535,0.06258133550484975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,1,0.011621333658695221
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,1,0.012671999633312225
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,7,0.012655999511480331
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,15,0.011706666400035223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,15,0.013349333157142004
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,31,0.011770666887362799
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,31,0.012858666479587555
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,63,0.011541333049535751
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,63,0.012666666259368261
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,127,0.011952000359694162
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,127,0.013866666704416275
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,255,0.015018666783968607
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,255,0.013002666334311167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,511,0.01357866699496905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,511,0.014762666076421738
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,1023,0.0145066666106383
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,1023,0.01544533297419548
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,2047,0.015498666713635126
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,8191,0.01942933350801468
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,2047,0.017210666090250015
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,4095,0.017130666722853977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,4095,0.018165333817402523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,8191,0.02090666691462199
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,16383,0.03762666632731756
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,16383,0.02629866699377696
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,32767,0.06005866825580597
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,32767,0.04348800083001455
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,65535,0.10617599884668986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,65535,0.06715733309586842
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,1,0.011488000551859537
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,1,0.012991999586423239
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,7,0.013023999830087027
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,3,0.011407999942700068
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,3,0.012250666817029318
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,7,0.011498666057984034
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,15,0.011445333560307821
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,15,0.013151999562978745
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,31,0.011434666812419891
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,31,0.012949333836634954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,63,0.011472000430027643
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,63,0.012576000144084295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,127,0.011183999478816986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,127,0.013450667262077332
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,255,0.012506666282812754
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,255,0.0138026662170887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,511,0.012890666723251343
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,511,0.014192000031471252
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,1023,0.014085333794355392
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,1023,0.015024000157912573
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,2047,0.015008000036080679
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,2047,0.01624533285697301
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,4095,0.017797333498795826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,4095,0.018976000448067982
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,8191,0.020981334149837494
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,8191,0.0234400009115537
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,16383,0.030000001192092896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,16383,0.03348266581694285
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,32767,0.04725866516431173
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,32767,0.0525546669960022
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,65535,0.08381866415341695
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,65535,0.09733333190282185
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,1,0.011578666667143503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,15,0.011802667131026586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,1,0.012618667135636011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,15,0.012800000607967377
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,3,0.011658667276302973
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,3,0.012351999680201212
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,7,0.011642667154471079
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,7,0.012479999413092932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,31,0.011546666423479715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,31,0.01246400053302447
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,63,0.0116799995303154
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,63,0.012650666137536367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,127,0.011957333733638128
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,127,0.01333333303531011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,255,0.012485332787036896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,255,0.013552000125249227
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,511,0.013034666577974955
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,511,0.014479999740918478
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,1023,0.014122666170199713
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,1023,0.015856000284353893
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,2047,0.016143999993801117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,2047,0.016490666816631954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,8191,0.023904000719388325
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,4095,0.018730666488409042
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,4095,0.01941866676012675
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,8191,0.021898667017618816
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,16383,0.03108799954255422
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,16383,0.03461333364248276
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,32767,0.051216001311937966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,32767,0.05413866539796194
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,1,0.012906666845083237
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,65535,0.08447999755541484
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,65535,0.09699733058611552
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,1,0.011786667009194693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,3,0.011781333635250727
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,3,0.012549333274364471
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,7,0.011829332758982977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,7,0.012746666868527731
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,15,0.01146666705608368
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,15,0.012885333349307379
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,31,0.011685332904259363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,31,0.01333333303531011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,63,0.011717333147923151
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,63,0.012858666479587555
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,127,0.011957333733638128
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,127,0.013658666362365087
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,255,0.013034666577974955
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,255,0.014655999839305878
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,511,0.014090667168299357
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,511,0.014736000448465347
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,1023,0.015253332753976187
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,1023,0.01646399994691213
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,2047,0.016330666840076447
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,2047,0.01708799973130226
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,4095,0.018538666268189747
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,4095,0.02012266715367635
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,8191,0.021946666141351063
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,8191,0.02462933212518692
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,16383,0.03811199963092804
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,16383,0.03533866753180822
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,32767,0.06012799839178721
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,32767,0.0621919979651769
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,65535,0.1074720025062561
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,65535,0.10123733679453532
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,1,0.011834666132926941
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,1,0.013183999806642532
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,3,0.01201066623131434
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,3,0.012874666601419449
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,7,0.011999999483426413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,7,0.012890666723251343
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,15,0.011952000359694162
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,15,0.013162666310866674
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,31,0.011850666254758835
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,31,0.012901333471139273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,63,0.012047999848922094
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,63,0.012826666235923767
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,127,0.012133333832025528
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,127,0.014378666877746582
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,255,0.013594667116800943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,255,0.015354666858911514
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,511,0.013973332941532135
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,511,0.015552000453074774
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,1023,0.015706667055686314
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,1023,0.016373333831628162
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,8191,0.03711466739575068
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,2047,0.017242666333913803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,2047,0.017535999417304993
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,4095,0.01966933285196622
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,32767,0.10241066416104634
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,4095,0.020453333854675293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,8191,0.02722666660944621
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,16383,0.059248000383377075
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,16383,0.04228266576925913
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,1,0.013066666821638743
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,32767,0.07559466858704884
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,65535,0.18918399016062418
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,65535,0.13075199723243713
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,fp8,1,0.013882666826248169
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,3,0.012944000462690989
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,fp8,3,0.013855999956528345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,7,0.01302933320403099
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,fp8,7,0.013493333011865616
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,15,0.01328533391157786
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,fp8,15,0.013514666507641474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,31,0.013050666699806849
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,fp8,31,0.013749333719412485
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,63,0.013066666821638743
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,255,0.015184000134468079
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,fp8,63,0.013749333719412485
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,127,0.013280000537633896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,fp8,511,0.018800000349680584
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,fp8,127,0.014864000181357065
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,fp8,255,0.01551466683546702
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,511,0.016672000288963318
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,1023,0.02184533327817917
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,fp8,1023,0.023018665611743927
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,2047,0.037231999138991036
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,fp8,2047,0.03486400097608566
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,8191,0.11197333534558614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,fp8,8191,0.10151466727256775
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,4095,0.06320533156394958
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,fp8,4095,0.058304001887639366
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,1,0.01350933313369751
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,fp8,1,0.014106666048367819
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,3,0.012991999586423239
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,fp8,3,0.014138666292031607
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,7,0.013461332768201828
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,fp8,7,0.014202666779359182
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,fp8,15,0.014277332772811254
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,15,0.01320533330241839
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,63,0.01369599997997284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,fp8,63,0.01414399966597557
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,31,0.013327999661366144
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,fp8,31,0.013951999445756277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,127,0.01381333296497663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,fp8,127,0.01569066693385442
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,255,0.015024000157912573
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,fp8,255,0.016117333124081295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,511,0.018101333330074947
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,1023,0.036277333895365395
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,fp8,1023,0.02479466547568639
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,fp8,511,0.018751999984184902
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,4095,0.10275733470916748
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,2047,0.05843733251094818
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,fp8,4095,0.06605866551399231
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,fp8,2047,0.04155199974775314
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,8191,0.18982932964960733
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,1,0.017349333812793095
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,fp8,1,0.01800000046690305
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,fp8,8191,0.11546666423479716
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,3,0.017312000195185345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,fp8,3,0.01820266619324684
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,fp8,7,0.018005333840847015
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,7,0.017093333105246227
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,15,0.01721599946419398
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,fp8,15,0.01833600054184596
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,31,0.017407999684413273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,fp8,31,0.01821333294113477
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,63,0.017514667163292568
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,fp8,127,0.02123733361562093
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,fp8,63,0.017994667092959087
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,255,0.02184533327817917
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,fp8,255,0.02252800017595291
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,127,0.01809599995613098
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,511,0.038319999972979225
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,fp8,511,0.028864001234372456
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,2047,0.10337600111961365
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,1023,0.060991997520128884
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,fp8,1023,0.04516266783078512
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,fp8,2047,0.07055999835332234
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,4095,0.18980266650517783
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,fp8,4095,0.11821867028872173
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,1,0.023786666492621105
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,8191,0.36462398370107013
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,fp8,8191,0.21299733718236288
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,fp8,1,0.025663999219735462
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,fp8,3,0.025807999074459076
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,3,0.024005333582560223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,7,0.024080000817775726
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,fp8,7,0.025727999707063038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,15,0.024090667565663654
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,fp8,15,0.026165333886941273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,31,0.02430933217207591
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,fp8,31,0.025888000925381977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,63,0.024400000770886738
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,fp8,63,0.025578667720158894
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,127,0.024735999604066212
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,fp8,127,0.0306986669699351
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,255,0.04196266829967499
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,fp8,255,0.03555200000603994
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,1023,0.10454400380452473
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,fp8,511,0.051370665431022644
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,fp8,1023,0.07413866619269054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,511,0.06273599962393443
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,2047,0.18986666202545166
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,fp8,2047,0.12343999743461609
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,1,0.010794666906197866
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,4095,0.36451733112335205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,fp8,4095,0.21579732497533163
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,3,0.010901333143313726
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,3,0.012351999680201212
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,7,0.010874666273593903
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,1,0.012234666695197424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,8191,0.7078613440195719
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,15,0.01139733319481214
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,7,0.011717333147923151
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,15,0.012005332857370377
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,fp8,8191,0.4030239979426066
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,31,0.011221333096424738
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,31,0.01181866725285848
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,63,0.011029332876205444
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,63,0.012453333785136541
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,127,0.011402666568756104
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,127,0.013157332936922709
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,255,0.012378666549921036
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,255,0.013823999712864557
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,511,0.013663999736309052
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,511,0.014597332725922266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,1023,0.014730667074521383
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,1023,0.01581866666674614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,2047,0.016309333344300587
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,2047,0.018394666413466137
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,4095,0.020618667205174763
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,16383,0.047728002071380615
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,4095,0.02310933421055476
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,8191,0.02871999889612198
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,8191,0.03301866600910822
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,16383,0.05232533315817515
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,32767,0.08333866794904073
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,32767,0.09701866904894511
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,65535,0.1530346671740214
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,65535,0.18556799491246542
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,1,0.01126933346192042
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,1,0.012138667205969492
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,15,0.011226666470368704
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,3,0.011445333560307821
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,3,0.011962667107582092
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,7,0.011391999820868174
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,7,0.01240533341964086
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,15,0.012096000214417776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,31,0.011429333438475927
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,31,0.012229333321253458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,63,0.011503999431928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,63,0.011957333733638128
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,127,0.011413333316644033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,127,0.013408000270525614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,255,0.012800000607967377
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,255,0.014032000054915747
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,511,0.014074667046467463
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,511,0.015647999942302704
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,1023,0.01545599972208341
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,1023,0.01613333324591319
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,2047,0.016927999754746754
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,8191,0.030000001192092896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,2047,0.018645333747069042
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,4095,0.020981334149837494
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,4095,0.023376000424226124
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,8191,0.03329066683848699
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,16383,0.0491946687301
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,32767,0.0844533344109853
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,16383,0.05304533243179321
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,32767,0.09469866752624512
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,65535,0.15336533387502035
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,1,0.011610666910807291
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,65535,0.17222400506337485
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,1,0.012485332787036896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,3,0.011546666423479715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,3,0.012506666282812754
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,31,0.012560000022252401
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,7,0.011503999431928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,63,0.01145600030819575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,7,0.012901333471139273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,15,0.011503999431928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,127,0.014389333625634512
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,255,0.01310933381319046
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,15,0.012741333494583765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,31,0.011541333049535751
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,63,0.012639999389648438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,127,0.011578666667143503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,255,0.015354666858911514
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,511,0.014597332725922266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,511,0.015791999797026317
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,1023,0.015893333901961643
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,1023,0.016757333030303318
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,2047,0.017925333231687546
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,2047,0.01915733392039935
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,4095,0.024447999894618988
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,16383,0.05985066791375478
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,4095,0.02384000023206075
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,8191,0.03709866603215536
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,8191,0.034117333590984344
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,16383,0.07169599831104279
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,32767,0.12818666299184164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,32767,0.10079999764760335
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,3,0.01209066684047381
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,65535,0.2394933303197225
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,1,0.011973333855470022
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,65535,0.18464533487955728
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,1,0.013210666676362356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,3,0.013466666142145792
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,7,0.01198400060335795
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,7,0.01340266689658165
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,15,0.012085333466529846
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,15,0.013034666577974955
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,31,0.012256000190973282
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,31,0.013376000026861826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,63,0.012106666962305704
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,63,0.013306666165590286
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,127,0.01231466606259346
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,511,0.01651200031240781
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,127,0.01443733274936676
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,255,0.013295999417702356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,255,0.014560000350077948
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,511,0.015397333850463232
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,1023,0.01617066686352094
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,1023,0.016965333372354507
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,8191,0.058464000622431435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,2047,0.018383999665578205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,2047,0.019653332730134327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,4095,0.037263999382654824
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,4095,0.026202666262785595
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,8191,0.04251199960708618
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,16383,0.10265599687894185
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,16383,0.07477866609891255
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,32767,0.18925867478052774
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,32767,0.11313066879908244
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,65535,0.3667733271916707
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,65535,0.21023466189702353
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,1,0.014544000228246054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,fp8,1,0.015423999478419622
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,3,0.014357333381970724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,fp8,3,0.015605332950750986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,7,0.01471466695268949
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,fp8,7,0.015610666324694952
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,15,0.014479999740918478
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,fp8,15,0.015765332927306492
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,31,0.01431999976436297
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,fp8,31,0.015311999867359797
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,63,0.014501333236694336
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,fp8,63,0.015589332828919092
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,127,0.01479999969402949
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,fp8,127,0.016666666915019352
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,fp8,255,0.01754666616519292
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,255,0.016943999876578648
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,511,0.019205333044131596
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,fp8,511,0.020346666375796
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,1023,0.037248000502586365
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,fp8,1023,0.026320000489552815
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,2047,0.0590826670328776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,fp8,2047,0.04424533247947693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,1,0.01775466650724411
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,fp8,1,0.018911999960740406
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,3,0.01786133274435997
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,4095,0.10379200180371602
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,fp8,4095,0.06925333539644878
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,fp8,3,0.018992000569899876
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,7,0.017701332767804463
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,fp8,7,0.018895999838908512
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,15,0.01794133335351944
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,fp8,15,0.018981333822011948
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,63,0.018160000443458557
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,fp8,63,0.01904533306757609
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,127,0.018079999834299088
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,31,0.018186666071414948
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,fp8,127,0.022053333620230358
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,255,0.022944000860055287
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,fp8,255,0.023605334262053173
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,fp8,31,0.018570666511853535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,511,0.04141333450873693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,fp8,511,0.03089066594839096
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,1023,0.06201066573460897
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,fp8,1023,0.04699199895064036
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,fp8,4095,0.11902933319409688
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,4095,0.19174933433532715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,1,0.02439466615517934
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,fp8,1,0.02651199946800868
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,fp8,2047,0.07123200098673503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,2047,0.10524800419807434
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,3,0.024325333535671234
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,7,0.024373332659403484
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,fp8,7,0.02649066597223282
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,15,0.024666666984558105
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,fp8,3,0.02678400029738744
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,fp8,15,0.02641066660483678
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,63,0.024533333877722423
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,31,0.0249493345618248
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,fp8,63,0.026367999613285065
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,127,0.025792000194390614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,fp8,127,0.031018666923046112
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,fp8,31,0.026682667434215546
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,255,0.04358933369318644
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,fp8,255,0.03559466699759165
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,511,0.06374933322270711
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,fp8,511,0.05385600030422211
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,fp8,2047,0.12548800309499106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,2047,0.19151999553044638
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,4095,0.36689066886901855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,fp8,4095,0.21954667568206787
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,fp8,1023,0.07684800028800964
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,1023,0.1050933301448822
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,1,0.037130666275819145
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,3,0.03749333322048187
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,fp8,1,0.040463998913764954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,fp8,3,0.040287998815377556
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,7,0.03743999948104223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,fp8,7,0.04078399886687597
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,31,0.037402667105197906
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,fp8,31,0.040549332896868386
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,63,0.03884266565243403
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,fp8,63,0.04030400017897288
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,127,0.04614933331807455
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,fp8,127,0.05385600030422211
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,15,0.03782933453718821
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,fp8,15,0.04027733455101649
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,fp8,255,0.06260266900062561
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,255,0.06474666794141133
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,1023,0.19364267587661743
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,fp8,1023,0.13292266925175986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,511,0.11042666435241699
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,fp8,511,0.08703999718030293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,2047,0.36371199289957684
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,fp8,2047,0.22914133469263712
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,1,0.020469332734743755
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,4095,0.7104427019755045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,fp8,4095,0.4129013220469157
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,fp8,3,0.02197333425283432
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,7,0.020117333779732387
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,fp8,1,0.02182399978240331
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,3,0.020303999384244282
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,fp8,7,0.021957332889238994
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,fp8,15,0.02204799900452296
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,15,0.020576000213623047
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,31,0.020560000091791153
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,fp8,31,0.02163200080394745
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,63,0.020634666085243225
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,fp8,63,0.021909333765506744
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,127,0.020949333906173706
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,fp8,127,0.02456533412138621
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,255,0.025637333591779072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,fp8,255,0.02664000044266383
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,511,0.043178667624791466
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,fp8,511,0.03306133300065994
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,1023,0.06579733391602834
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,fp8,1023,0.05117333432038625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,1,0.026704000929991405
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,fp8,1,0.02787200113137563
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,3,0.026869334280490875
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,7,0.026650667190551758
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,fp8,3,0.02773866554101308
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,fp8,15,0.02826133370399475
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,fp8,7,0.02810666710138321
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,31,0.026869334280490875
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,15,0.02696000039577484
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,fp8,31,0.027978666126728058
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,63,0.027056001126766205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,fp8,63,0.027749332288901012
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,127,0.02937600016593933
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,fp8,127,0.03274133304754893
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,255,0.0443146675825119
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,fp8,255,0.03811733424663544
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,511,0.06878933310508728
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,fp8,511,0.055455997586250305
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,1,0.03976000100374222
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,fp8,1,0.04247466723124186
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,1023,0.10970667004585266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,fp8,1023,0.07813866436481476
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,3,0.039359999199708305
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,fp8,3,0.04241600135962168
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,15,0.039818666875362396
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,7,0.03967999915281931
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,fp8,7,0.04274666806062063
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,fp8,15,0.04252799848715464
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,63,0.0418453315893809
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,31,0.04004266609748205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,fp8,63,0.042805333932240806
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,fp8,31,0.04234133164087931
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,127,0.0498879998922348
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,fp8,127,0.0536053329706192
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,fp8,255,0.06637866795063019
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,255,0.07004266480604808
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,511,0.11313600341478984
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,fp8,511,0.09115200241406758
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,1,0.06542933483918507
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,fp8,1,0.0699946681658427
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,3,0.06526400148868561
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,fp8,3,0.06985066831111908
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,fp8,1023,0.13590932885805765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,1023,0.19818667570749918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,fp8,15,0.07014399766921997
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,15,0.06566933294137318
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,7,0.06518933176994324
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,fp8,7,0.0703359991312027
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,31,0.06772799789905548
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,fp8,31,0.07011733452479045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,127,0.07815999786059062
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,fp8,127,0.09241599837938945
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,255,0.11849600076675415
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,fp8,255,0.1083626647790273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,1023,0.3704640070597331
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,fp8,1023,0.24729067087173462
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,63,0.07099199791749318
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,fp8,63,0.07254933317502339
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,fp8,511,0.15727466344833374
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,511,0.20297066370646158
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,1,0.03489600121974945
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,fp8,1,0.03664533297220866
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,3,0.03514666606982549
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,fp8,3,0.03676266719897588
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,7,0.0349386657277743
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,fp8,7,0.03663466622432073
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,31,0.03505599995454153
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,fp8,31,0.036858665446440377
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,63,0.035455999275048576
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,fp8,63,0.036992001036802925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,fp8,15,0.03667200108369192
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,15,0.035205334424972534
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,127,0.03743999948104223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,fp8,127,0.040474665661652885
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,255,0.05459199845790863
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,fp8,255,0.047839999198913574
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,1,0.046096002062161766
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,511,0.07814933359622955
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,fp8,511,0.06502399841944377
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,fp8,1,0.047685335079828896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,3,0.04555733501911163
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,7,0.045994664231936135
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,fp8,3,0.04795200129350027
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,fp8,7,0.047882666190465294
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,15,0.04640000065167745
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,fp8,15,0.04836800197760264
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,31,0.04614399870236715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,63,0.04891733328501383
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,fp8,31,0.04819199939568838
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,fp8,63,0.047914668917655945
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,127,0.05905599892139435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,fp8,127,0.0611413319905599
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,255,0.07884799937407176
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,fp8,255,0.0730506678422292
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,1,0.07271466652552287
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,511,0.1220960021018982
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,fp8,511,0.09702933828035991
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,fp8,1,0.077824001510938
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,3,0.07338666419188182
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,7,0.07299200197060902
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,fp8,3,0.0790773332118988
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,fp8,7,0.0774239997069041
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,15,0.07306666672229767
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,fp8,31,0.07829333345095317
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,fp8,15,0.0787360022465388
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,63,0.07904000083605449
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,31,0.07657066484292348
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,fp8,63,0.08265066643555959
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,127,0.08780266841252644
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,fp8,127,0.10065600275993347
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,255,0.12619733810424805
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,fp8,255,0.11872532963752747
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,1,0.12377599875132243
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,fp8,1,0.13154133160909018
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,3,0.12496532996495564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,fp8,511,0.1691946585973104
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,511,0.21368533372879028
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,fp8,3,0.13083733121554056
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,7,0.12523200114568075
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,fp8,7,0.13100799918174744
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,fp8,15,0.13175466656684875
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,15,0.12711466352144876
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,31,0.13176533579826355
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,fp8,63,0.13821867108345032
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,127,0.1452959974606832
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,fp8,127,0.17113600174585977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,63,0.1349120040734609
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,fp8,31,0.13740266362826029
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,1,0.011349332829316458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,255,0.21925334135691324
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,3,0.012485332787036896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,3,0.011434666812419891
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,7,0.011477333803971609
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,1,0.012309333930412928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,15,0.011477333803971609
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,fp8,255,0.2016800045967102
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,15,0.01258133351802826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,31,0.011493333925803503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,31,0.012389333297808966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,63,0.011461333682139715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,127,0.011381333072980246
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,127,0.013546666751305262
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,255,0.01267733300725619
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,255,0.014645333091417948
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,511,0.014197333405415217
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,511,0.015487999965747198
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,1023,0.016021333634853363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,7,0.01257066677014033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,1023,0.018533332894245785
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,2047,0.02040533348917961
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,2047,0.022917332748572033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,63,0.012538666526476542
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,4095,0.02890666574239731
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,4095,0.032773333291212715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,8191,0.04744000236193339
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,511,0.3939093351364136
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,fp8,511,0.29393066962560016
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,8191,0.05227733155091604
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,16383,0.0828959991534551
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,16383,0.0957973301410675
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,32767,0.1516320010026296
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,1,0.01227733368674914
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,32767,0.18369599183400473
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,65535,0.29263466596603394
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,3,0.011530666301647821
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,3,0.012421333541472753
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,15,0.011823999385039011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,7,0.011744000017642975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,7,0.01227733368674914
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,15,0.012554666648308435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,31,0.01138666644692421
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,31,0.012586666891972223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,63,0.01166933278242747
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,63,0.012517333030700684
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,127,0.011866666376590729
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,127,0.013594667116800943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,255,0.01313599944114685
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,255,0.01544533297419548
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,511,0.014869333555301031
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,511,0.016255999604860943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,1023,0.017130666722853977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,1023,0.018458666900793713
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,2047,0.021018666525681812
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,2047,0.0232640008131663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,4095,0.0306986669699351
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,4095,0.03339733431736628
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,8191,0.05338133374849955
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,1,0.011610666910807291
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,8191,0.04923733572165171
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,65535,0.3542826573053996
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,16383,0.0844106674194336
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,16383,0.09482133388519287
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,32767,0.15453333655993143
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,32767,0.17244267463684082
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,65535,0.2935946583747864
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,1,0.012106666962305704
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,65535,0.3282613356908162
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,1,0.013269333789745966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,3,0.0120319997270902
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,3,0.013274667163689932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,7,0.012128000458081564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,31,0.0120319997270902
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,7,0.013418667018413544
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,15,0.01228800043463707
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,63,0.01358933374285698
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,15,0.013258667041858038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,31,0.013429333766301474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,63,0.012335999558369318
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,127,0.012170666207869848
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,127,0.015642666568358738
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,255,0.014138666292031607
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,255,0.015791999797026317
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,511,0.014730667074521383
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,511,0.016496000190575916
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,1023,0.017029333859682083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,1023,0.018687999496857326
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,2047,0.021295999487241108
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,2047,0.0240639994541804
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,4095,0.03803733239571253
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,4095,0.033999999364217125
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,8191,0.06169599791367849
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,8191,0.05832533538341522
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,16383,0.10884267091751099
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,16383,0.1002453366915385
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,32767,0.20989867051442465
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,32767,0.18393067518870035
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,65535,0.3472213347752889
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,1,0.012426666915416718
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,65535,0.37348798910776776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,1,0.013477332890033722
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,3,0.012367999802033106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,3,0.013280000537633896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,7,0.012367999802033106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,7,0.013455999394257864
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,15,0.01249066616098086
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,15,0.013786666095256805
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,31,0.012335999558369318
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,31,0.013242666920026144
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,63,0.01211200033624967
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,63,0.013530666629473368
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,127,0.012560000022252401
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,127,0.01522133375207583
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,255,0.013994666437307993
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,255,0.016506666938463848
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,511,0.01551466683546702
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,511,0.01721599946419398
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,4095,0.05794133245944977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,1023,0.01749333366751671
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,1023,0.01904533306757609
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,8191,0.1021066705385844
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,2047,0.036373332142829895
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,2047,0.024746666351954143
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,4095,0.04196799794832865
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,8191,0.06654933094978333
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,16383,0.18922666708628336
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,16383,0.13117333253224692
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,32767,0.36572265625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,32767,0.2348853349685669
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,65535,0.7163200378417969
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,65535,0.39476267496744794
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,1,0.03886933376391729
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,fp8,1,0.039103999733924866
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,fp8,3,0.03926933308442434
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,3,0.03825066735347112
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,7,0.038480001191298165
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,fp8,7,0.03892799963553747
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,15,0.03851733356714249
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,fp8,15,0.0391146664818128
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,31,0.03914133210976919
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,fp8,31,0.03879466652870178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,63,0.043749332427978516
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,127,0.04762133459250132
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,fp8,63,0.038933334251244865
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,fp8,127,0.05125333368778229
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,255,0.06862399975458781
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,fp8,255,0.06481066842873891
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,1,0.062314664324124656
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,fp8,1,0.062650665640831
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,3,0.06192533175150553
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,fp8,3,0.062277331948280334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,7,0.06201600035031637
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,fp8,7,0.06238399942715963
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,15,0.0625546673933665
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,fp8,15,0.06294399996598561
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,31,0.06649066507816315
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,fp8,31,0.0627040018637975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,63,0.07095466554164886
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,fp8,63,0.06612800061702728
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,127,0.07692799965540568
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,fp8,127,0.08559466401735942
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,1,0.11614933609962463
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,255,0.11612266302108765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,fp8,255,0.1039573351542155
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,fp8,1,0.11366400122642517
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,fp8,3,0.11343999703725179
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,7,0.11521066228548686
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,3,0.11492799719174702
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,fp8,7,0.11348266402880351
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,15,0.12461333473523457
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,fp8,15,0.114656001329422
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,fp8,31,0.1216159959634145
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,31,0.12619733810424805
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,fp8,63,0.12477333347002666
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,63,0.12752532958984375
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,1,0.21103467543919882
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,fp8,127,0.1579039990901947
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,127,0.13797866304715475
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,fp8,1,0.20556267102559408
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,255,0.2098133365313212
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,fp8,255,0.18917334079742432
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,3,0.2111306587855021
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,fp8,3,0.20573866367340088
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,7,0.2254613240559896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,fp8,7,0.20618132750193277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,fp8,15,0.21693867444992065
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,15,0.22766933838526407
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,31,0.23041599988937378
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,fp8,31,0.22343466679255167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,63,0.23105067014694214
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,fp8,63,0.22241065899531046
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,127,0.24918399254480997
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,fp8,127,0.2900533278783162
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,255,0.3873973290125529
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,fp8,255,0.3507839838663737
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,1,0.06611200173695882
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,fp8,1,0.066021333138148
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,3,0.06539200246334076
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,fp8,3,0.06577066580454509
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,7,0.06568000217278798
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,fp8,7,0.06595199803511302
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,31,0.07429333527882893
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,fp8,31,0.06875200072924297
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,15,0.0682239979505539
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,fp8,15,0.06592000027497609
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,fp8,63,0.0765173335870107
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,127,0.08218666911125183
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,fp8,127,0.09051199754079182
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,fp8,1,0.11388799548149109
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,1,0.11237866679827373
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,63,0.07436266541481018
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,3,0.11205866932868958
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,7,0.11214933792750041
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,fp8,3,0.11257599790891011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,fp8,7,0.11322133739789327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,15,0.11896533767382304
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,fp8,15,0.11414399743080139
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,31,0.12289599577585857
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,fp8,31,0.11986133456230164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,63,0.1253600021203359
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,fp8,63,0.12282133102416992
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,127,0.13924800356229147
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,fp8,127,0.15397333105405173
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,1,0.21849066019058228
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,fp8,1,0.21555733680725098
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,3,0.22686400016148886
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,fp8,3,0.212719996770223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,7,0.2360480030377706
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,fp8,7,0.21566933393478394
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,fp8,15,0.23060266176859537
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,31,0.23892800013224283
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,15,0.23597866296768188
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,fp8,31,0.2330133318901062
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,63,0.24307199319203696
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,fp8,63,0.23253333568572998
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,127,0.2595733404159546
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,1,0.4227733214696248
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,fp8,127,0.3004106680552165
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,fp8,1,0.39909334977467853
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,3,0.4398719867070516
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,fp8,3,0.4113759994506836
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,7,0.4461386601130168
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,fp8,7,0.4211466709772746
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,15,0.44307732582092285
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,fp8,15,0.4251999855041504
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,fp8,31,0.42626134554545086
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,31,0.4429866472880046
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,1,0.011626667032639185
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,3,0.012479999413092932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,1,0.012858666479587555
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,7,0.01166933278242747
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,3,0.011648000528415045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,7,0.012794667234023413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,15,0.01146666705608368
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,15,0.012597333639860153
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,31,0.011871999750534693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,31,0.012682666381200155
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,fp8,63,0.4318133195241292
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,63,0.011706666400035223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,63,0.012815999488035837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,127,0.01250133290886879
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,63,0.4467039903004964
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,127,0.013872000078360239
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,255,0.014896000425020853
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,511,0.01728533332546552
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,511,0.01498666654030482
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,1023,0.01931200052301089
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,2047,0.028042666614055634
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,2047,0.03186666717131933
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,4095,0.04561600089073181
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,4095,0.05086933573087057
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,8191,0.08291199803352356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,8191,0.09493866562843323
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,255,0.013194666554530462
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,16383,0.14962666233380637
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,16383,0.18397865692774454
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,127,0.4837813377380371
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,32767,0.2903573314348857
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,1023,0.02219199885924657
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,32767,0.3524320125579834
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,1,0.012319999436537424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,1,0.01350933313369751
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,7,0.012997332960367203
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,3,0.012191999703645706
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,3,0.01302933320403099
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,15,0.012191999703645706
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,15,0.013461332768201828
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,31,0.012448000411192576
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,31,0.013061333447694778
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,63,0.012250666817029318
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,127,0.014858666807413101
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,63,0.013050666699806849
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,127,0.012821332861979803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,255,0.014373333503802618
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,255,0.015461333096027374
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,7,0.012159999459981918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,511,0.01602666700879733
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,511,0.017866666118303936
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,1023,0.02073066681623459
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,1023,0.02219199885924657
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,fp8,127,0.5642240047454834
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,2047,0.029066666960716248
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,2047,0.032111999889214836
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,4095,0.04948266843954722
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,4095,0.051589335004488625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,8191,0.08374933401743571
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,8191,0.09383466839790344
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,16383,0.15593600273132324
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,16383,0.1702559987703959
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,1,0.012117333710193634
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,1,0.013295999417702356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,7,0.01228800043463707
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,32767,0.32496533791224164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,3,0.012432000289360682
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,3,0.013717333475748697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,7,0.013855999956528345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,15,0.0124746672809124
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,15,0.013503999759753546
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,31,0.012272000312805176
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,31,0.013717333475748697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,63,0.012549333274364471
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,63,0.01368533323208491
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,32767,0.29127466678619385
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,127,0.012565333396196365
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,127,0.014858666807413101
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,255,0.014032000054915747
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,255,0.015418666104475657
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,511,0.015599999576807022
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,511,0.018581333259741466
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,1023,0.020224000016848247
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,1023,0.022858666876951855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,2047,0.03733866661787033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,2047,0.03359466542800268
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,4095,0.05979733169078827
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,4095,0.057402665416399636
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,8191,0.10797866185506184
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,8191,0.09981866677602132
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,16383,0.19538132349650064
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,16383,0.18198400735855103
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,3,0.012624000509579977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,1,0.01250133290886879
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,7,0.012597333639860153
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,32767,0.370410680770874
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,1,0.013658666362365087
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,32767,0.3503520091374715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,3,0.013845333208640417
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,7,0.01351999988158544
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,15,0.01249066616098086
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,15,0.013690666606028875
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,31,0.01246400053302447
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,31,0.013717333475748697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,63,0.012560000022252401
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,255,0.014101333916187286
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,63,0.013994666437307993
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,127,0.012624000509579977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,511,0.01646399994691213
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,127,0.015279999623696009
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,255,0.016010666886965435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,511,0.018650667121013004
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,1023,0.03514666606982549
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,1023,0.0242399995525678
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,4095,0.06465066472689311
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,2047,0.057520002126693726
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,8191,0.18876266479492188
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,8191,0.12988799810409546
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,2047,0.04101866732041041
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,4095,0.10044800241788228
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,16383,0.36552000045776367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,16383,0.20710400740305582
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,1,0.012133333832025528
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,1,0.013066666821638743
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,32767,0.39527467886606854
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,32767,0.7143466472625732
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,7,0.012159999459981918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,3,0.01219733307758967
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,3,0.013189333180586496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,7,0.012928000340859095
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,15,0.01232533281048139
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,15,0.012938667088747025
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,31,0.01227733368674914
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,31,0.012906666845083237
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,63,0.012186666329701742
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,63,0.013157332936922709
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,127,0.012213333199421564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,127,0.014325333138306936
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,255,0.01358933374285698
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,255,0.015605332950750986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,511,0.01544533297419548
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,511,0.017594666530688603
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,2047,0.032138665517171226
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,1023,0.01959466685851415
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,1023,0.02248000105222066
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,2047,0.029872000217437744
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,4095,0.0487360010544459
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,8191,0.08473066488901775
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,4095,0.053344001372655235
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,8191,0.09946667154630025
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,16383,0.15285866459210715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,16383,0.1850773294766744
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,7,0.011952000359694162
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,1,0.012613333761692047
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,1,0.013482666263977686
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,3,0.012250666817029318
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,3,0.013568000247081121
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,7,0.013669333110253016
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,15,0.012560000022252401
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,15,0.013514666507641474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,31,0.011930666863918304
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,31,0.013925333817799887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,63,0.012495999534924826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,63,0.01350933313369751
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,127,0.012437333663304647
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,127,0.01452800010641416
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,255,0.013760000467300415
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,255,0.015722667177518208
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,511,0.015749332805474598
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,511,0.018090666582187016
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,1023,0.020773333807786305
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,1023,0.022613334159056347
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,2047,0.03670933345953623
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,8191,0.10378666718800862
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,2047,0.03456533451875051
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,4095,0.06121066709359487
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,4095,0.0582826683918635
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,8191,0.09900266925493877
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,16383,0.2011893391609192
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,3,0.01250133290886879
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,3,0.01370666672786077
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,16383,0.1809813380241394
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,1,0.012560000022252401
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,1,0.013722666849692663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,7,0.012741333494583765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,7,0.013957332819700241
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,15,0.012714666624863943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,15,0.013669333110253016
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,31,0.01239466667175293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,31,0.013989333063364029
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,63,0.012762666990359625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,63,0.014042666802803675
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,127,0.012874666601419449
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,511,0.01700266698996226
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,127,0.015461333096027374
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,1023,0.035429333647092186
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,255,0.01586666703224182
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,255,0.015850666910409927
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,511,0.018618666877349217
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,1023,0.023792001108328503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,2047,0.05735466877619425
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,2047,0.041050667564074196
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,4095,0.10070932904879253
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,4095,0.06576000154018402
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,8191,0.18766933679580688
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,8191,0.11331733067830403
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,1,0.011333333949247995
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,1,0.01259200026591619
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,16383,0.20922666788101196
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,16383,0.3637760082880656
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,3,0.011418666690587997
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,3,0.012047999848922094
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,7,0.011450666934251785
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,7,0.012330666184425354
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,15,0.011514666179815928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,15,0.012634667257467905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,31,0.011440000186363855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,31,0.012752000242471695
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,63,0.011317333827416102
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,63,0.012906666845083237
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,127,0.01137599969903628
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,127,0.014949332922697067
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,255,0.013647999614477158
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,255,0.01451733335852623
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,511,0.01381333296497663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,511,0.014426667243242264
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,1023,0.013946666071812311
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,1023,0.015008000036080679
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,2047,0.015130666395028433
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,2047,0.016063999384641647
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,4095,0.016122666498025257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,4095,0.01717866708834966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,8191,0.018053332964579265
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,8191,0.02000533292690913
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,16383,0.022885332504908245
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,16383,0.025120000044504803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,32767,0.03258133431275686
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,32767,0.035216001172860466
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,65535,0.05138133466243744
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,65535,0.05663999915122986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,1,0.011221333096424738
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,1,0.012901333471139273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,3,0.01138666644692421
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,3,0.012602667013804117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,7,0.011461333682139715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,7,0.012346666306257248
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,15,0.011530666301647821
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,15,0.012896000097195307
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,31,0.01126933346192042
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,31,0.012842666357755661
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,63,0.012762666990359625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,63,0.012944000462690989
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,127,0.011407999942700068
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,127,0.014581333845853806
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,255,0.013839999834696451
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,255,0.014789332946141561
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,511,0.013557333499193192
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,511,0.015109332899252573
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,1023,0.013951999445756277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,1023,0.015311999867359797
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,2047,0.01524266724785169
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,2047,0.016858667135238647
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,4095,0.01708799973130226
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,4095,0.017727999637524288
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,8191,0.018842666099468868
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,8191,0.020874666670958202
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,16383,0.023973333338896435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,16383,0.02573866645495097
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,32767,0.03404266635576884
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,32767,0.03600533306598663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,65535,0.05412800113360087
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,65535,0.059792002042134605
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,1,0.011519999553759893
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,1,0.01301866645614306
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,15,0.013045333325862885
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,3,0.011674666156371435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,3,0.012762666990359625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,7,0.011952000359694162
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,7,0.012458667159080505
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,15,0.011733333269755045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,31,0.011503999431928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,31,0.013189333180586496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,63,0.011472000430027643
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,63,0.012442667037248611
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,127,0.011616000284751257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,127,0.014858666807413101
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,255,0.013765333841244379
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,255,0.015050667027632395
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,511,0.01393066719174385
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,511,0.015664000064134598
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,1023,0.014725333700577417
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,1023,0.016293333222468693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,2047,0.01672533278663953
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,2047,0.01732800031701724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,4095,0.017237332959969837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,4095,0.018496000518401463
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,8191,0.01916266605257988
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,8191,0.020773333807786305
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,16383,0.0242399995525678
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,16383,0.025978667040665943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,65535,0.062458669145902
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,32767,0.039034667114416756
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,32767,0.0386613334218661
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,65535,0.0620000014702479
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,1,0.01129066695769628
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,1,0.012810666114091873
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,3,0.01137599969903628
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,3,0.012346666306257248
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,7,0.011391999820868174
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,7,0.012847999731699625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,15,0.011296000331640244
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,15,0.012645332763592402
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,31,0.011231999844312668
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,31,0.012367999802033106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,63,0.011215999722480774
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,63,0.012917333592971167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,127,0.011370666325092316
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,127,0.01471466695268949
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,255,0.013839999834696451
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,255,0.014831999937693277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,511,0.013744000345468521
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,511,0.014789332946141561
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,1023,0.015013333410024643
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,1023,0.016074666132529575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,2047,0.01618133361140887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,2047,0.01710933322707812
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,4095,0.01855466639002164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,4095,0.019882666567961376
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,16383,0.03443199892838796
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,8191,0.021509334444999695
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,8191,0.024288001159826916
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,16383,0.030368000268936157
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,32767,0.04828266799449921
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,32767,0.05389333268006643
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,3,0.011509332805871964
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,65535,0.08429333567619324
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,65535,0.09366400043169658
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,1,0.011424000064531961
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,1,0.012746666868527731
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,3,0.012863999853531519
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,7,0.011648000528415045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,7,0.012773333738247553
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,15,0.011530666301647821
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,15,0.012618667135636011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,31,0.01139733319481214
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,31,0.01249066616098086
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,63,0.011605333536863327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,63,0.012789333860079447
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,127,0.01166933278242747
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,127,0.01498666654030482
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,255,0.01381333296497663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,255,0.014869333555301031
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,511,0.013834666460752487
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,511,0.015658666690190632
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,1023,0.015605332950750986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,1023,0.01706133286158244
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,2047,0.01729600007335345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,2047,0.01782400036851565
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,4095,0.019050666441520054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,4095,0.020442667106787365
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,8191,0.022304000953833263
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,8191,0.024693332612514496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,16383,0.03136000037193298
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,16383,0.03499199946721395
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,32767,0.05035733183224996
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,32767,0.057333335280418396
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,65535,0.0848640004793803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,65535,0.10193066795667012
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,1,0.011792000383138657
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,1,0.012362666428089142
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,3,0.011744000017642975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,3,0.012736000120639801
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,7,0.01179733375708262
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,7,0.012879999975363413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,15,0.011637333780527115
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,63,0.012805332740147909
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,15,0.012693333129088083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,31,0.011850666254758835
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,31,0.012437333663304647
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,63,0.011727999895811081
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,127,0.011861333002646765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,127,0.015301333119471868
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,255,0.014175999909639359
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,255,0.015696000307798386
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,511,0.014602666099866232
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,511,0.016229332735141117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,1023,0.015989333391189575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,1023,0.017024000485738117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,2047,0.01741333305835724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,2047,0.01834133391578992
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,4095,0.019530666371186573
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,4095,0.0206133338312308
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,8191,0.02483733246723811
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,8191,0.025429333249727886
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,16383,0.038736000657081604
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,16383,0.03685333331425985
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,32767,0.06159466505050659
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,32767,0.060415998101234436
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,65535,0.1290773351987203
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,65535,0.10267200072606404
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,1,0.013173333058754602
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,fp8,3,0.014053333550691605
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,fp8,1,0.013562666873137156
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,3,0.013194666554530462
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,7,0.013173333058754602
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,fp8,7,0.013647999614477158
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,15,0.012879999975363413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,fp8,15,0.013536000003417334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,31,0.013210666676362356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,fp8,31,0.013760000467300415
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,63,0.013151999562978745
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,fp8,63,0.013845333208640417
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,127,0.014069333672523499
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,fp8,127,0.014949332922697067
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,255,0.01470400020480156
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,fp8,511,0.018298666924238205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,fp8,255,0.015989333391189575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,511,0.016783999900023144
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,1023,0.021989333132902782
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,fp8,1023,0.023168000082174938
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,2047,0.03719999889532725
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,fp8,2047,0.0336053321758906
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,4095,0.05993066728115082
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,8191,0.12846933801968893
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,fp8,4095,0.058090666929880776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,fp8,8191,0.1018239955107371
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,1,0.013317332913478216
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,fp8,1,0.014192000031471252
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,3,0.013429333766301474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,fp8,3,0.014090667168299357
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,fp8,7,0.014106666048367819
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,7,0.013199999928474426
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,fp8,15,0.014170666535695394
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,15,0.013264000415802002
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,31,0.01314666618903478
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,127,0.013818666338920593
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,fp8,31,0.014170666535695394
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,63,0.013274667163689932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,fp8,63,0.014106666048367819
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,fp8,127,0.01533866673707962
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,511,0.017792000124851864
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,255,0.015605332950750986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,fp8,255,0.01621866722901662
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,1023,0.03630933413902918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,fp8,511,0.01889066646496455
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,fp8,1023,0.02439466615517934
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,4095,0.10168000062306722
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,fp8,4095,0.06645333270231883
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,2047,0.0580266664425532
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,fp8,2047,0.04229333500067393
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,8191,0.1886026660601298
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,fp8,8191,0.11730666955312093
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,1,0.017125333348910015
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,fp8,1,0.018405333161354065
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,3,0.01738133281469345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,fp8,3,0.018320000420014065
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,7,0.01711999997496605
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,15,0.017263999829689663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,fp8,7,0.017898666361967724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,fp8,15,0.01830400029818217
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,31,0.017082666357358296
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,fp8,31,0.018426666657129925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,63,0.017317333569129307
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,fp8,63,0.018373332917690277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,127,0.017509333789348602
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,fp8,127,0.021216000119845074
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,511,0.03844266633192698
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,255,0.021312000850836437
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,fp8,255,0.022837333381175995
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,fp8,511,0.029989334444204967
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,1023,0.060271998246510826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,fp8,1023,0.04458666841189066
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,2047,0.10355732838312785
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,fp8,2047,0.07004266480604808
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,4095,0.1905333399772644
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,fp8,4095,0.1186133325099945
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,3,0.010746666540702185
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,8191,0.3636266787846883
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,1,0.01098666712641716
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,3,0.012554666648308435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,fp8,8191,0.21270400285720825
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,1,0.012186666329701742
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,7,0.01097600037852923
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,7,0.011786667009194693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,63,0.012309333930412928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,15,0.010874666273593903
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,31,0.010863999525705973
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,15,0.01249066616098086
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,31,0.012730666746695837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,63,0.01121066634853681
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,127,0.010911999891201654
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,127,0.014266667266686758
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,255,0.013280000537633896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,255,0.014544000228246054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,511,0.014639999717473984
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,511,0.015909332782030106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,1023,0.01590399940808614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,1023,0.016890666137139004
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,2047,0.017029333859682083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,2047,0.01932266727089882
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,4095,0.021344001094500225
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,4095,0.023728000621000927
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,8191,0.030000001192092896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,8191,0.0340693344672521
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,16383,0.04821333289146423
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,65535,0.15317866206169128
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,16383,0.053557331363360085
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,32767,0.08428800106048584
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,1,0.012213333199421564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,32767,0.09641066193580627
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,65535,0.18386665980021158
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,1,0.011285333583752314
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,3,0.01116266722480456
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,3,0.01210133358836174
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,7,0.01102399950226148
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,7,0.013882666826248169
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,15,0.011312000453472137
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,15,0.012266666938861212
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,31,0.011274666835864386
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,31,0.01231466606259346
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,63,0.011152000476916632
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,63,0.012186666329701742
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,127,0.011535999675591787
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,127,0.014842666685581207
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,255,0.013701333353916803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,255,0.01598400001724561
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,511,0.015034666905800501
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,511,0.01658133293191592
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,1023,0.016176000237464905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,1023,0.017504000415404636
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,2047,0.017866666118303936
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,2047,0.02000533292690913
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,4095,0.02161066730817159
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,4095,0.02458133300145467
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,8191,0.031343999008337654
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,8191,0.03440000116825104
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,16383,0.0505973349014918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,16383,0.05533866584300995
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,32767,0.08557333548863728
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,32767,0.10192533334096272
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,65535,0.1544533371925354
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,65535,0.18261333306630453
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,7,0.011733333269755045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,1,0.011541333049535751
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,1,0.012485332787036896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,15,0.01302933320403099
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,31,0.011616000284751257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,3,0.01156266654531161
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,3,0.012768000364303589
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,7,0.012543999900420507
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,15,0.011855999628702799
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,31,0.012634667257467905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,63,0.011477333803971609
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,63,0.012917333592971167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,127,0.011866666376590729
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,127,0.015029333531856537
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,255,0.014389333625634512
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,255,0.01551466683546702
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,511,0.015717333803574245
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,511,0.01684800038735072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,1023,0.016688000410795212
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,1023,0.017984000345071156
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,2047,0.01815466706951459
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,2047,0.019893333315849304
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,4095,0.02292266736427943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,4095,0.02498133232196172
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,8191,0.038346665600935616
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,8191,0.03522133330504099
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,16383,0.06111466884613037
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,16383,0.05889600018660227
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,32767,0.10658133029937744
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,32767,0.10155199964841206
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,65535,0.20362667242685953
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,65535,0.1856266657511393
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,1,0.014682666709025701
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,fp8,1,0.01533866673707962
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,3,0.014645333091417948
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,fp8,3,0.015439999600251516
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,7,0.014362666755914688
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,fp8,7,0.01524266724785169
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,15,0.014256000518798828
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,fp8,15,0.015552000453074774
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,31,0.01443733274936676
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,fp8,31,0.015423999478419622
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,63,0.014416000495354334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,fp8,63,0.015477333217859268
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,127,0.014831999937693277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,fp8,127,0.017018667111794155
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,255,0.016693333784739178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,fp8,255,0.017664000391960144
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,511,0.02011200040578842
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,fp8,511,0.02013333390156428
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,1023,0.03782933453718821
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,fp8,1023,0.025936000049114227
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,2047,0.05946133534113566
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,fp8,2047,0.043237333496411644
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,1,0.017653333644072216
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,fp8,1,0.018853332847356796
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,3,0.01781333362062772
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,4095,0.10342933734258015
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,fp8,3,0.019194666296243668
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,fp8,4095,0.06884799897670746
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,7,0.017898666361967724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,fp8,7,0.019018666197856266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,15,0.017674667139848072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,fp8,15,0.018917333334684372
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,fp8,31,0.019194666296243668
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,fp8,63,0.01930133377512296
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,31,0.017701332767804463
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,127,0.01841066653529803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,fp8,127,0.02203733225663503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,63,0.017866666118303936
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,255,0.021946666141351063
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,fp8,255,0.023498666783173878
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,511,0.04078399886687597
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,fp8,511,0.02939733366171519
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,1023,0.06228800117969513
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,fp8,1023,0.046821330984433494
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,4095,0.19166932503382364
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,fp8,4095,0.12147200107574463
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,1,0.024383999407291412
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,2047,0.10419199864069621
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,fp8,2047,0.07170666754245758
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,fp8,1,0.026389333109060924
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,fp8,3,0.026485333840052288
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,3,0.024314666787783306
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,7,0.024698667228221893
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,fp8,7,0.026341333985328674
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,31,0.024693332612514496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,fp8,31,0.026591998835404713
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,63,0.024671999116738636
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,15,0.0245919997493426
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,fp8,63,0.026533332963784535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,127,0.025807999074459076
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,fp8,15,0.02621866762638092
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,fp8,127,0.03091199944416682
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,255,0.0431573341290156
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,fp8,255,0.03463999927043915
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,511,0.06438933312892914
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,2047,0.19271999597549438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,fp8,511,0.052570665876070656
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,fp8,2047,0.12475732962290446
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,1023,0.10520000259081523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,fp8,1023,0.07662400106589
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,4095,0.36635732650756836
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,fp8,4095,0.21740265687306723
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,1,0.019930666933457058
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,fp8,1,0.022074667116006214
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,fp8,3,0.021685334543387096
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,3,0.019999999552965164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,15,0.019765333582957584
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,31,0.020074666788180668
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,fp8,15,0.02141333371400833
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,fp8,31,0.021568000316619873
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,63,0.020186666399240494
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,fp8,63,0.021920000513394673
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,fp8,7,0.021738665799299877
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,127,0.020432000358899433
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,7,0.020746666938066483
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,fp8,127,0.024186665813128155
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,255,0.024117333193620045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,fp8,255,0.026309333741664886
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,511,0.04264000058174133
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,fp8,511,0.03417599946260452
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,1023,0.06564799944559734
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,fp8,1023,0.050426666935284935
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,fp8,1,0.028031999866167705
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,1,0.026949333647886913
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,3,0.026021334032217663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,fp8,3,0.028501334289709728
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,7,0.026191999514897663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,fp8,7,0.0284853329261144
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,15,0.026389333109060924
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,31,0.02622399975856145
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,fp8,15,0.028586665789286297
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,63,0.026320000489552815
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,fp8,31,0.02900800108909607
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,127,0.02826133370399475
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,fp8,63,0.028560000161329906
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,fp8,127,0.033226666351159416
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,255,0.046298667788505554
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,fp8,255,0.03788800040880839
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,511,0.06807999809583028
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,fp8,511,0.056074668963750206
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,fp8,1,0.04230933388074239
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,1,0.03977066775163015
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,3,0.03932266682386398
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,fp8,3,0.04197866717974345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,fp8,1023,0.07969599962234497
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,7,0.039077334105968475
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,1023,0.11103467146555583
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,fp8,7,0.042090664307276406
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,15,0.03934400031963984
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,fp8,15,0.04251733422279358
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,31,0.03939199944337209
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,fp8,31,0.041989331444104515
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,fp8,63,0.042863999803860985
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,fp8,127,0.05182399849096934
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,63,0.04197866717974345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,255,0.07112533350785573
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,fp8,255,0.06595199803511302
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,127,0.04945066571235657
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,511,0.11241066455841064
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,fp8,511,0.09045867125193278
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,1,0.034058667719364166
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,fp8,1,0.03573866685231527
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,1023,0.19599467515945435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,fp8,1023,0.1360213359196981
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,3,0.03446399917205175
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,fp8,3,0.03590933233499527
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,fp8,7,0.035530666510264076
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,7,0.03398933261632919
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,15,0.034058667719364166
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,fp8,31,0.03610666592915853
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,63,0.03448000053564707
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,fp8,63,0.036176001032193504
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,127,0.03687999894221624
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,fp8,127,0.03973866750796636
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,fp8,15,0.03558400024970373
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,31,0.03402666747570038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,255,0.05379733443260193
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,fp8,255,0.04684266448020935
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,511,0.07589866717656453
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,fp8,511,0.06365866462389629
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,1,0.04563199977080027
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,fp8,1,0.04827733337879181
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,3,0.04539733131726583
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,fp8,3,0.04850133260091146
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,7,0.0455626646677653
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,fp8,7,0.049082666635513306
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,15,0.04574933151404063
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,31,0.0459146648645401
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,fp8,31,0.04829333225886027
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,fp8,15,0.04872000217437744
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,63,0.04801600178082784
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,fp8,63,0.048709332942962646
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,127,0.057775999108950295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,fp8,127,0.058602665861447654
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,255,0.07789866626262665
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,fp8,255,0.07387733459472656
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,1,0.07141866783301036
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,511,0.12115200360616048
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,fp8,511,0.10006933410962422
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,fp8,1,0.07696000238259633
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,3,0.07165333131949107
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,fp8,3,0.0779306689898173
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,7,0.07190933326880138
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,fp8,7,0.07735466460386912
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,15,0.07230400045712788
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,fp8,31,0.07725333174069722
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,fp8,63,0.0800799975792567
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,fp8,15,0.07787733276685078
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,63,0.0796800007422765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,31,0.07588266829649608
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,1,0.01138666644692421
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,127,0.08582933743794759
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,1,0.01249066616098086
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,fp8,127,0.10036266843477885
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,3,0.011354666203260422
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,255,0.12449066837628682
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,7,0.012133333832025528
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,3,0.01249066616098086
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,fp8,255,0.11795733372370402
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,15,0.011535999675591787
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,7,0.012975999464591345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,31,0.011450666934251785
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,31,0.012650666137536367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,fp8,511,0.1678346594174703
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,15,0.012245333443085352
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,63,0.011503999431928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,511,0.21158933639526367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,63,0.012410666793584824
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,127,0.011525332927703857
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,127,0.013573333621025085
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,255,0.012842666357755661
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,511,0.014815999815861383
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,511,0.016741332908471424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,1023,0.019402666638294857
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,2047,0.021418665846188862
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,2047,0.024170666933059692
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,255,0.016688000410795212
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,4095,0.030207999050617218
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,4095,0.03483733286460241
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,8191,0.0484746644894282
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,1023,0.017136000096797943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,8191,0.05313600103060404
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,16383,0.08366400003433228
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,16383,0.09659733374913533
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,32767,0.15341333548227945
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,32767,0.18522665898005167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,65535,0.2906559904416402
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,1,0.011744000017642975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,65535,0.3515626589457194
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,1,0.012639999389648438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,3,0.011551999797423681
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,3,0.012666666259368261
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,7,0.01163200040658315
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,7,0.012858666479587555
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,15,0.011359999577204386
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,15,0.013487999637921652
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,31,0.011690666278203329
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,31,0.012639999389648438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,63,0.011621333658695221
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,63,0.012618667135636011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,127,0.011893333246310553
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,127,0.013669333110253016
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,255,0.01309866706530253
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,255,0.016704000532627106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,511,0.015658666690190632
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,511,0.017680000513792038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,1023,0.017898666361967724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,1023,0.019786667078733444
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,2047,0.02237333357334137
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,2047,0.024858665963013966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,4095,0.03143466760714849
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,4095,0.034645333886146545
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,8191,0.05190399785836538
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,8191,0.05676800012588501
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,16383,0.0860533316930135
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,16383,0.10129066308339436
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,32767,0.1562399963537852
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,32767,0.18376533190409342
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,65535,0.2945760091145833
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,1,0.012037333101034164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,65535,0.3496319850285848
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,1,0.013594667116800943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,3,0.011765333513418833
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,3,0.013594667116800943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,7,0.011978667229413986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,7,0.013514666507641474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,15,0.012026666353146235
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,15,0.01341333364446958
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,31,0.012202666451533636
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,31,0.01322666679819425
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,63,0.01201066623131434
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,63,0.013658666362365087
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,127,0.01246400053302447
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,127,0.014607999473810196
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,255,0.013872000078360239
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,2047,0.022495999932289124
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,255,0.017263999829689663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,511,0.016106666376193363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,4095,0.039146666725476585
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,511,0.01770666614174843
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,1023,0.01788266624013583
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,8191,0.058687999844551086
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,16383,0.10988799730936687
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,1023,0.020058666666348774
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,2047,0.02508266766866048
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,4095,0.03641066700220108
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,8191,0.06198399762312571
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,16383,0.10237333178520203
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,32767,0.19503466288248697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,32767,0.1834933360417684
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,65535,0.3729066848754883
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,65535,0.3534613450368245
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,1,0.03811199963092804
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,fp8,1,0.038975998759269714
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,3,0.0383840004603068
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,fp8,3,0.03922666609287262
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,7,0.038149334490299225
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,fp8,7,0.038906666139761605
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,15,0.038762666285037994
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,fp8,15,0.03916800022125244
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,31,0.03818133225043615
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,63,0.04313066601753235
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,fp8,63,0.039333333571751915
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,127,0.04729066789150238
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,255,0.06821866830190022
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,fp8,255,0.06470400094985962
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,fp8,31,0.039520000418027244
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,fp8,127,0.050255998969078064
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,1,0.06409066418806712
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,fp8,1,0.06296533346176147
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,3,0.06414933502674103
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,fp8,3,0.0628959983587265
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,7,0.06478400031725566
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,fp8,7,0.06313066681226094
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,15,0.06397866706053416
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,fp8,31,0.06338666876157124
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,31,0.06913066903750102
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,fp8,15,0.06256533165772755
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,63,0.07190933326880138
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,fp8,63,0.06905599931875865
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,127,0.07773866752783458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,fp8,127,0.0890826682249705
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,1,0.11488533020019531
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,fp8,1,0.11241599917411804
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,fp8,255,0.10667733351389568
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,255,0.11557867129643758
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,3,0.11538133025169373
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,fp8,3,0.11457600196202596
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,7,0.1162559986114502
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,fp8,7,0.11428800225257874
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,15,0.1222826639811198
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,fp8,15,0.11306132872899373
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,31,0.1264586647351583
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,63,0.12811199824015299
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,fp8,31,0.1204159955183665
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,fp8,63,0.12479999661445618
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,127,0.13739200433095297
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,fp8,127,0.15904532869656882
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,255,0.20827200015385947
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,fp8,255,0.18825600544611612
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,1,0.06543999910354614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,fp8,1,0.06571733454863231
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,3,0.06514133512973785
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,fp8,3,0.06659733255704244
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,7,0.06584000090758006
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,fp8,15,0.06601066887378693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,fp8,7,0.06568000217278798
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,15,0.06727466483910878
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,31,0.07299200197060902
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,fp8,31,0.06778666873772939
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,63,0.07363733152548473
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,fp8,63,0.07423466444015503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,127,0.08196266492207845
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,fp8,1,0.11276266972223918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,1,0.11550933122634888
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,fp8,127,0.08921600381533305
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,7,0.11831466356913249
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,fp8,3,0.11220266421635945
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,fp8,7,0.11380267143249512
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,15,0.12786133090655008
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,fp8,15,0.11345066626866658
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,31,0.1274186670780182
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,fp8,31,0.12377066413561504
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,3,0.11621866623560588
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,63,0.1288106640179952
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,127,0.14033066232999167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,fp8,127,0.15924266974131265
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,fp8,63,0.12586133678754172
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,fp8,1,0.21310933430989584
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,3,0.22016000747680664
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,fp8,3,0.2132586638132731
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,7,0.23540266354878744
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,fp8,7,0.2179786761601766
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,15,0.2390026648839315
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,fp8,15,0.2277173399925232
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,1,0.21794132391611734
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,1,0.011823999385039011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,1,0.012815999488035837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,3,0.011616000284751257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,3,0.013834666460752487
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,7,0.011754666765530905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,fp8,31,0.23054399092992148
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,31,0.23733333746592203
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,7,0.012789333860079447
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,15,0.011701333026091257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,31,0.011695999652147293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,15,0.012917333592971167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,63,0.011663999408483505
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,31,0.012522666404644648
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,63,0.012901333471139273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,127,0.011594666788975397
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,127,0.013807999591032663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,255,0.012821332861979803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,255,0.01488000030318896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,511,0.015103999525308609
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,511,0.01735466718673706
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,1023,0.019365333020687103
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,1023,0.022053333620230358
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,2047,0.027999999622503918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,2047,0.03183466692765554
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,4095,0.046207999189694725
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,4095,0.050666665037473045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,8191,0.08156266808509827
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,63,0.24116265773773193
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,8191,0.09320533275604248
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,16383,0.15174933274586996
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,16383,0.18204800287882486
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,32767,0.2872533400853475
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,1,0.013253333667914072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,32767,0.35123201211293537
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,3,0.012256000190973282
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,3,0.013744000345468521
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,1,0.011941333611806234
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,7,0.011866666376590729
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,7,0.013151999562978745
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,15,0.012144000579913458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,15,0.01340266689658165
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,31,0.012058666596810022
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,fp8,63,0.23327465852101645
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,31,0.013141332815090815
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,fp8,127,0.3022080063819885
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,63,0.012037333101034164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,63,0.013701333353916803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,127,0.26016533374786377
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,127,0.01240533341964086
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,127,0.014746667196353277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,255,0.013631999492645264
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,511,0.015578666081031164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,1023,0.02277333289384842
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,1023,0.01958400011062622
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,255,0.015184000134468079
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,2047,0.033157333731651306
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,4095,0.04900800188382467
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,4095,0.054671997825304665
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,8191,0.08502933382987976
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,8191,0.09887466828028361
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,16383,0.15455999970436096
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,511,0.018309333672126133
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,16383,0.18231467405954996
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,32767,0.2921173373858134
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,1,0.012117333710193634
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,2047,0.029509333272775013
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,32767,0.3469333251317342
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,1,0.013274667163689932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,7,0.0138026662170887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,3,0.012453333785136541
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,3,0.013434667140245438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,7,0.012383999923865
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,15,0.01250133290886879
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,15,0.013701333353916803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,63,0.01339200014869372
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,31,0.012309333930412928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,31,0.013663999736309052
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,63,0.012597333639860153
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,127,0.012853333105643591
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,127,0.015365333606799444
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,255,0.014208000153303146
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,255,0.01533866673707962
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,511,0.01575999955336253
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,2047,0.03688533355792364
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,511,0.018207999567190807
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,1023,0.02128533273935318
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,1023,0.02290133386850357
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,2047,0.03583999971548716
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,4095,0.06052266558011373
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,4095,0.057258665561676025
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,8191,0.12599999705950418
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,8191,0.09883200128873189
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,16383,0.23757867018381754
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,16383,0.18272000551223755
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,1,0.011999999483426413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,32767,0.3469280004501343
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,1,0.013487999637921652
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,32767,0.46034665902455646
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,3,0.01209066684047381
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,3,0.013290667285521826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,7,0.012133333832025528
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,7,0.013056000073750814
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,15,0.012186666329701742
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,15,0.01312000056107839
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,31,0.012357333054145178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,31,0.013658666362365087
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,63,0.012602667013804117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,63,0.013471999516089758
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,127,0.012768000364303589
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,127,0.014271999398867289
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,511,0.017701332767804463
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,1023,0.020256000260512035
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,255,0.013621332744757334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,255,0.015354666858911514
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,511,0.016447999825080235
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,1023,0.022490667800108593
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,2047,0.029690665503342945
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,2047,0.03205333401759466
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,4095,0.04915200173854828
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,4095,0.053216000398000084
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,8191,0.08468799789746602
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,8191,0.09488000472386678
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,16383,0.1530026694138845
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,1,0.012522666404644648
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,16383,0.17319466670354208
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,32767,0.29151999950408936
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,32767,0.3277440071105957
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,1,0.013397333522637686
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,3,0.01246400053302447
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,3,0.013354666531085968
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,7,0.012080000092585882
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,7,0.013797332843144735
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,15,0.012181332955757776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,15,0.01369599997997284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,31,0.012330666184425354
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,31,0.013397333522637686
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,255,0.013973332941532135
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,63,0.012282667060693106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,63,0.013471999516089758
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,127,0.012367999802033106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,127,0.015173333386580149
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,255,0.015370666980743408
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,511,0.015589332828919092
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,511,0.018005333840847015
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,1023,0.02130666623512904
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,1023,0.022533332308133442
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,2047,0.03714133302370707
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,4095,0.05949866771697998
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,2047,0.03472000112136205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,4095,0.05726400017738342
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,8191,0.12566933035850525
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,8191,0.09890133142471313
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,16383,0.20200000206629434
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,16383,0.18280533949534097
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,32767,0.3877120018005371
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,1,0.013738666971524557
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,1,0.012714666624863943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,32767,0.3487093448638916
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,3,0.012517333030700684
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,3,0.013877333452304205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,7,0.012522666404644648
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,7,0.013674666484196981
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,15,0.012538666526476542
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,15,0.013877333452304205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,31,0.012709333250919977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,31,0.013621332744757334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,63,0.01259200026591619
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,63,0.013989333063364029
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,127,0.01292266696691513
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,127,0.015333333363135656
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,255,0.014373333503802618
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,255,0.01594666639963786
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,511,0.016613333175579708
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,511,0.018565333137909572
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,1023,0.03549866626660029
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,1023,0.02426133304834366
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,2047,0.05658133327960968
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,2047,0.041264000038305916
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,4095,0.10123733679453532
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,4095,0.06478400031725566
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,8191,0.18854933977127075
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,8191,0.1151146690050761
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,16383,0.2084160049756368
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,16383,0.3633493185043335
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,1,0.016154666741689045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,32767,0.394538680712382
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,1,0.01798933371901512
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,3,0.01658133293191592
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,32767,0.7130719820658366
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,3,0.017952000101407368
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,7,0.016544000556071598
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,7,0.017786666750907898
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,15,0.016447999825080235
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,15,0.017850667238235474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,31,0.016629333297411602
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,31,0.018288000176350277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,63,0.016528000434239704
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,63,0.017877332866191864
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,127,0.01670933390657107
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,127,0.020928000410397846
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,255,0.02062400057911873
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,255,0.02218666672706604
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,511,0.03772799919048945
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,511,0.030218665798505146
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,1023,0.05871999760468801
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,1023,0.04394666850566864
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,2047,0.1018399993578593
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,2047,0.06868266562620799
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,4095,0.18927466869354248
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,4095,0.11639466881752014
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,8191,0.36164267857869464
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,8191,0.21098132928212485
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,1,0.011648000528415045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,16383,0.3953973452250163
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,1,0.012448000411192576
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,16383,0.7125866413116455
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,3,0.0116799995303154
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,3,0.012479999413092932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,7,0.011391999820868174
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,32767,0.7715679804484049
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,7,0.012282667060693106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,15,0.011551999797423681
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,32767,1.5298986434936523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,15,0.012576000144084295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,127,0.011541333049535751
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,31,0.011648000528415045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,31,0.01257066677014033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,63,0.011546666423479715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,127,0.013440000514189402
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,63,0.012416000167528788
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,255,0.012335999558369318
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,255,0.01331199953953425
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,511,0.012479999413092932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,511,0.013679999858140945
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,1023,0.01332266628742218
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,1023,0.014170666535695394
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,2047,0.014218666901191076
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,2047,0.015119999647140503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,4095,0.015344000111023584
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,4095,0.016352000335852306
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,8191,0.017765333255132038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,8191,0.0191040001809597
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,16383,0.022416000564893086
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,16383,0.02443733314673106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,32767,0.03190399954716364
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,131071,0.0993280013402303
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,32767,0.03473600000143051
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,65535,0.050704002380371094
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,65535,0.05589333176612854
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,131071,0.08890133102734883
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,1,0.011541333049535751
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,1,0.012693333129088083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,3,0.011333333949247995
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,3,0.012703999876976013
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,7,0.011370666325092316
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,7,0.012074666718641916
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,15,0.011440000186363855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,15,0.012543999900420507
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,31,0.01145600030819575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,31,0.012784000486135483
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,63,0.011525332927703857
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,63,0.013951999445756277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,127,0.01145600030819575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,127,0.013541333377361298
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,255,0.012261333564917246
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,255,0.013797332843144735
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,511,0.012997332960367203
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,511,0.014170666535695394
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,1023,0.013327999661366144
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,4095,0.017429333180189133
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,1023,0.015130666395028433
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,2047,0.014368000129858652
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,2047,0.016042667130629223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,4095,0.01646399994691213
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,8191,0.018645333747069042
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,8191,0.020319999506076176
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,16383,0.023354666928450268
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,16383,0.025648000339667004
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,32767,0.033887999753157295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,32767,0.035504000882307686
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,65535,0.05358933409055074
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,65535,0.058575997749964394
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,131071,0.09113599856694539
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,131071,0.10617599884668986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,1,0.011706666400035223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,1,0.012682666381200155
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,3,0.011535999675591787
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,3,0.01268799975514412
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,7,0.01156266654531161
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,7,0.012671999633312225
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,15,0.011770666887362799
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,15,0.012453333785136541
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,31,0.011557333171367645
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,31,0.012400000045696894
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,63,0.011781333635250727
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,63,0.0129120002190272
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,127,0.011610666910807291
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,511,0.012858666479587555
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,127,0.014240000396966934
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,255,0.012602667013804117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,255,0.014074667046467463
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,511,0.014943999548753103
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,1023,0.015381333728631338
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,1023,0.01393066719174385
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,2047,0.016095999628305435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,2047,0.017210666090250015
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,4095,0.017221332838137943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,4095,0.017557332913080852
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,8191,0.01876266673207283
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,8191,0.02033599962790807
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,16383,0.02829866607983907
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,16383,0.02622933437426885
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,32767,0.03893866638342539
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,32767,0.03753600021203359
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,1,0.012485332787036896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,65535,0.06107733150323232
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,65535,0.06222933530807495
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,131071,0.11559999982515971
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,131071,0.1076639990011851
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,1,0.011701333026091257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,3,0.011584000041087469
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,3,0.012597333639860153
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,7,0.011530666301647821
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,63,0.011594666788975397
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,7,0.013178666432698568
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,15,0.011642667154471079
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,15,0.012784000486135483
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,31,0.011866666376590729
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,31,0.012608000387748083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,63,0.01268799975514412
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,127,0.01163200040658315
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,127,0.013621332744757334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,255,0.012991999586423239
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,255,0.014917333920796713
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,511,0.013631999492645264
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,511,0.014981333166360855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,1023,0.014602666099866232
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,1023,0.01544533297419548
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,2047,0.016042667130629223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,2047,0.016773333152135212
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,4095,0.01720000058412552
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,4095,0.018063999712467194
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,32767,0.06026133398214976
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,8191,0.02057066683967908
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,65535,0.10548266768455505
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,8191,0.020853333175182343
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,16383,0.0372533326347669
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,16383,0.02621866762638092
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,32767,0.042992000778516136
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,65535,0.06656533479690552
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,131071,0.19553599754969278
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,131071,0.13371200362841287
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,1,0.011653333902359009
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,1,0.012527999778588613
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,3,0.011440000186363855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,3,0.012304000556468964
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,7,0.011519999553759893
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,7,0.0122079998254776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,15,0.011482667177915573
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,15,0.01231466606259346
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,31,0.011509332805871964
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,31,0.01257066677014033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,63,0.011413333316644033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,63,0.012634667257467905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,127,0.011535999675591787
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,127,0.01312000056107839
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,255,0.012469333906968435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,255,0.013823999712864557
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,511,0.013072000195582708
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,511,0.013839999834696451
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,1023,0.01404800017674764
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,1023,0.014650666465361914
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,2047,0.01524266724785169
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,2047,0.015882667154073715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,8191,0.02348266790310542
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,4095,0.017557332913080852
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,4095,0.018730666488409042
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,8191,0.02072000006834666
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,16383,0.029333333174387615
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,16383,0.03347733368476232
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,65535,0.09417600433031718
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,32767,0.04673600196838379
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,32767,0.0529120018084844
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,65535,0.08266666531562805
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,131071,0.15031466881434122
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,131071,0.17243200540542603
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,1,0.011370666325092316
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,1,0.012752000242471695
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,15,0.011424000064531961
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,3,0.01137599969903628
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,3,0.013007999708255133
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,7,0.0116799995303154
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,7,0.012608000387748083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,15,0.012485332787036896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,31,0.012784000486135483
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,31,0.011557333171367645
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,63,0.011690666278203329
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,63,0.012794667234023413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,127,0.011514666179815928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,127,0.014021333307027817
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,255,0.012671999633312225
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,255,0.013679999858140945
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,511,0.01293333371480306
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,511,0.015034666905800501
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,1023,0.014202666779359182
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,1023,0.016197333733240765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,2047,0.016789333273967106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,2047,0.01682666689157486
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,4095,0.018042666216691334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,4095,0.01987733319401741
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,8191,0.021317332983016968
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,8191,0.024165332317352295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,16383,0.0305173322558403
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,16383,0.034458667039871216
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,32767,0.05087999999523163
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,32767,0.05563200016816457
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,65535,0.08470933636029561
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,65535,0.1018933355808258
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,131071,0.15542933344841003
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,131071,0.18399999539057413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,1,0.011685332904259363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,1,0.012671999633312225
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,15,0.012682666381200155
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,3,0.011663999408483505
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,3,0.013088000317414602
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,7,0.011584000041087469
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,7,0.01250133290886879
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,15,0.01190399999419848
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,31,0.011727999895811081
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,31,0.01293333371480306
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,63,0.011567999919255575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,63,0.012725333372751871
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,127,0.011792000383138657
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,127,0.014096000542243322
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,255,0.012842666357755661
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,255,0.014752000570297241
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,511,0.013621332744757334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,511,0.014954666296641031
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,1023,0.015247999380032221
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,1023,0.016373333831628162
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,2047,0.01624533285697301
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,2047,0.01749333366751671
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,4095,0.018672000616788864
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,4095,0.020261333634455998
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,8191,0.0220320001244545
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,8191,0.02463999887307485
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,16383,0.037605332831541695
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,16383,0.03566399961709976
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,32767,0.0639573335647583
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,32767,0.0591786652803421
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,65535,0.10964799920717876
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,3,0.01192533348997434
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,65535,0.10094400246938069
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,7,0.011871999750534693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,131071,0.24241065979003906
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,131071,0.18319465716679892
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,1,0.011695999652147293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,1,0.01312000056107839
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,3,0.012917333592971167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,7,0.013173333058754602
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,15,0.011941333611806234
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,15,0.012794667234023413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,31,0.011973333855470022
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,31,0.0129120002190272
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,63,0.012026666353146235
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,63,0.013088000317414602
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,127,0.01184533288081487
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,127,0.014485333114862442
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,255,0.013397333522637686
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,255,0.015098666151364645
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,511,0.01381333296497663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,511,0.015402667224407196
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,4095,0.019189332922299702
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,1023,0.015466666469971338
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,1023,0.016735999534527462
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,2047,0.01643199970324834
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,2047,0.017407999684413273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,4095,0.020389333367347717
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,8191,0.03738666574160258
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,8191,0.025413334369659424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,16383,0.05913066864013672
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,16383,0.04215999941031138
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,32767,0.10327999790509541
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,65535,0.13116799791653952
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,32767,0.0670826683441798
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,65535,0.1889973282814026
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,131071,0.3662986755371094
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,1,0.013023999830087027
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,131071,0.24403733015060425
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,1,0.014021333307027817
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,3,0.013434667140245438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,3,0.01339200014869372
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,7,0.013002666334311167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,15,0.013429333766301474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,15,0.01350933313369751
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,7,0.013365333278973898
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,31,0.013343999783198038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,31,0.014010666559139887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,63,0.01333333303531011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,63,0.013381333400805792
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,127,0.013839999834696451
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,127,0.015087999403476715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,255,0.015658666690190632
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,255,0.01588800052801768
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,511,0.017194667210181553
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,511,0.018383999665578205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,1023,0.022122666239738464
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,1023,0.02253866692384084
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,2047,0.03769599894682566
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,2047,0.03491200009981791
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,8191,0.10958400368690491
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,8191,0.09516266981760661
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,4095,0.07366933425267537
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,4095,0.05517866710821787
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,16383,0.24020800987879434
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,16383,0.17242666085561117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,1,0.01302933320403099
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,1,0.014192000031471252
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,3,0.013359999905029932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,15,0.01332266628742218
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,15,0.013807999591032663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,3,0.013999999811251959
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,7,0.01340266689658165
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,7,0.014058666924635569
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,63,0.013295999417702356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,31,0.013359999905029932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,31,0.014245333770910898
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,63,0.013978666315476099
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,127,0.01370666672786077
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,127,0.015520000209410986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,255,0.015018666783968607
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,255,0.0162773331006368
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,511,0.017957333475351334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,511,0.018757333358128864
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,1023,0.0367999995748202
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,1023,0.024522667129834492
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,2047,0.0581279993057251
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,2047,0.0422026664018631
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,4095,0.10139733552932739
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,4095,0.0662666658560435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,8191,0.1893706719080607
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,8191,0.11636799573898315
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,3,0.01714133347074191
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,16383,0.36396801471710205
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,3,0.017925333231687546
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,16383,0.20941332976023355
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,1,0.017152000218629837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,15,0.01739199956258138
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,7,0.017130666722853977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,1,0.018277333428462345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,31,0.018085333208243053
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,7,0.018320000420014065
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,15,0.018229333062966663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,31,0.01728533332546552
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,63,0.017349333812793095
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,63,0.018485333770513535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,127,0.017701332767804463
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,127,0.02164799968401591
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,255,0.022240000466505688
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,255,0.022624000906944275
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,511,0.03847466657559077
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,511,0.028229333460330963
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,1023,0.059989333152770996
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,1023,0.045109331607818604
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,2047,0.10268266995747884
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,2047,0.06919466455777486
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,4095,0.189626673857371
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,4095,0.11782933274904887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,8191,0.36160000165303546
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,1,0.023813332120577495
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,8191,0.2125493288040161
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,1,0.025445332129796345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,3,0.024005333582560223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,16383,0.39983999729156494
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,3,0.025402667621771496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,7,0.023999998966852825
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,7,0.025370667378107708
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,16383,0.7134666442871094
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,15,0.024031999210516613
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,15,0.025568000972270966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,31,0.0245919997493426
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,31,0.02554133286078771
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,63,0.024314666787783306
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,63,0.025349333882331848
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,127,0.02678400029738744
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,127,0.030378667016824085
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,255,0.04242133100827535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,255,0.035536001125971474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,511,0.06342933575312297
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,511,0.051514665285746254
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,1023,0.10434133807818095
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,1023,0.07472000022729237
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,2047,0.1897546648979187
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,2047,0.12205333511034648
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,4095,0.3622986475626628
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,4095,0.21625065803527832
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,1,0.010874666273593903
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,8191,0.40166934331258136
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,1,0.01201066623131434
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,3,0.011247999966144562
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,7,0.011061333119869232
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,3,0.012250666817029318
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,8191,0.7115840117136637
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,16383,0.7743840217590332
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,7,0.011866666376590729
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,15,0.01119999960064888
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,15,0.012005332857370377
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,16383,1.5239413579305012
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,31,0.011061333119869232
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,31,0.012448000411192576
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,63,0.010874666273593903
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,63,0.011829332758982977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,127,0.012149333953857422
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,127,0.01310933381319046
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,255,0.01239466667175293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,511,0.013631999492645264
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,255,0.013440000514189402
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,511,0.014592000593741735
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,1023,0.014655999839305878
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,1023,0.015130666395028433
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,2047,0.016085332880417507
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,2047,0.018186666071414948
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,4095,0.020282667130231857
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,16383,0.052042668064435325
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,4095,0.02271999915440877
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,8191,0.02845866729815801
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,8191,0.032602667808532715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,16383,0.04679466784000397
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,32767,0.08295466502507527
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,32767,0.09393067161242168
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,65535,0.1518400013446808
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,65535,0.1718026598294576
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,131071,0.28728000322977704
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,131071,0.3264799912770589
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,1,0.011237333218256632
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,1,0.011957333733638128
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,3,0.011173332730929056
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,3,0.012159999459981918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,7,0.011343999455372492
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,7,0.01228800043463707
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,15,0.011173332730929056
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,15,0.012421333541472753
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,127,0.01350933313369751
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,31,0.011296000331640244
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,31,0.012005332857370377
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,63,0.011173332730929056
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,63,0.01232533281048139
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,1023,0.015530666957298914
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,127,0.01129066695769628
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,1023,0.016458666572968166
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,255,0.012719999998807907
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,255,0.014346666634082794
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,511,0.013701333353916803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,511,0.015781333049138386
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,2047,0.01672533278663953
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,2047,0.018965333700180054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,4095,0.02109333376089732
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,4095,0.023728000621000927
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,8191,0.030080000559488933
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,8191,0.03373866776625315
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,16383,0.04943466683228811
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,16383,0.05478399991989136
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,32767,0.0860533316930135
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,32767,0.10101333260536194
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,65535,0.15382400155067444
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,65535,0.1819093426068624
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,131071,0.2949440081914266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,1,0.012202666451533636
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,131071,0.3500639994939168
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,1,0.012736000120639801
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,3,0.01163200040658315
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,3,0.01232533281048139
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,7,0.011717333147923151
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,7,0.012757333616415659
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,15,0.011503999431928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,15,0.012576000144084295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,31,0.011626667032639185
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,31,0.012682666381200155
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,63,0.011541333049535751
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,63,0.012618667135636011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,127,0.011722666521867117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,127,0.014607999473810196
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,255,0.01331199953953425
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,255,0.014805333067973455
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,511,0.01526933287580808
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,511,0.01621866722901662
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,1023,0.015919999529918034
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,1023,0.016906666258970898
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,2047,0.017231999586025875
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,2047,0.019258666783571243
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,4095,0.024373332659403484
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,4095,0.023845332364241283
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,8191,0.03800000001986822
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,8191,0.03497066597143809
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,16383,0.0606826643149058
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,16383,0.05856533348560333
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,65535,0.18421334028244019
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,32767,0.1272266705830892
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,32767,0.10083199540774028
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,65535,0.2030400037765503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,1,0.012047999848922094
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,131071,0.3893226782480876
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,1,0.013440000514189402
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,3,0.012165332833925882
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,131071,0.3476959864298503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,3,0.013338666409254074
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,7,0.011952000359694162
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,7,0.013125333935022354
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,15,0.011973333855470022
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,15,0.013013333082199097
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,31,0.011871999750534693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,31,0.013130666067202887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,63,0.012037333101034164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,63,0.013530666629473368
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,127,0.01221866657336553
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,127,0.014357333381970724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,255,0.01381333296497663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,255,0.014933332800865173
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,511,0.015376000354687372
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,511,0.016149333367745083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,1023,0.016458666572968166
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,1023,0.016821333517630894
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,2047,0.017984000345071156
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,8191,0.04223999877770742
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,2047,0.019413333386182785
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,4095,0.03702399879693985
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,4095,0.024906667570273083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,8191,0.05834133426348368
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,16383,0.10214933753013611
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,16383,0.06618133187294006
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,32767,0.18889600038528442
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,32767,0.11437867085138957
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,65535,0.36632001399993896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,65535,0.24473599592844644
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,131071,0.39422400792439777
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,131071,0.72597336769104
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,1,0.01471466695268949
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,fp8,1,0.015392000476519266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,3,0.014303999642531076
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,fp8,3,0.015013333410024643
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,7,0.014607999473810196
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,fp8,7,0.01562133307258288
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,15,0.014789332946141561
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,fp8,15,0.015178666760524115
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,31,0.014565333724021912
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,fp8,31,0.015392000476519266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,63,0.014554666976133982
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,fp8,63,0.015119999647140503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,127,0.015189333508412043
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,fp8,127,0.016885332763195038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,255,0.0173333336909612
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,fp8,255,0.01756799966096878
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,511,0.019343999524911244
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,1023,0.037733333806196846
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,fp8,511,0.019797333826621372
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,fp8,1023,0.025786665578683216
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,2047,0.05972800155480703
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,fp8,2047,0.04354133208592733
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,8191,0.19122666120529175
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,1,0.01764800027012825
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,fp8,8191,0.11538133025169373
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,4095,0.10384000341097514
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,fp8,4095,0.06794133285681407
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,fp8,1,0.01921066641807556
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,3,0.017738666385412216
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,fp8,3,0.019173332800467808
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,7,0.017786666750907898
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,fp8,7,0.0189280000825723
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,31,0.01806933308641116
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,fp8,31,0.019343999524911244
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,63,0.017887999614079792
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,15,0.01798933371901512
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,fp8,63,0.019088000059127808
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,fp8,15,0.018837332725524902
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,127,0.01828266680240631
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,fp8,127,0.022298666338125866
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,255,0.022469334304332733
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,fp8,255,0.023333333432674408
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,511,0.03941866755485535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,fp8,511,0.02938666691382726
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,1023,0.0617439995209376
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,fp8,1023,0.04696000119050344
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,4095,0.19245866934458414
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,fp8,4095,0.12109333276748657
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,2047,0.10505066315333049
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,fp8,2047,0.07216000060240428
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,8191,0.3651786645253499
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,1,0.024325333535671234
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,fp8,1,0.02628266563018163
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,fp8,8191,0.21397332350413004
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,3,0.02418133368094762
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,7,0.024234667420387268
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,15,0.02441066751877467
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,fp8,15,0.026202666262785595
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,fp8,3,0.026634665826956432
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,31,0.02455466737349828
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,fp8,7,0.026330667237440746
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,fp8,31,0.02603200078010559
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,63,0.02478400121132533
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,fp8,63,0.02624000112215678
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,127,0.025914666553338368
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,fp8,127,0.030794667700926464
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,255,0.04385599990685781
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,fp8,255,0.03469866762558619
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,511,0.06412266691525777
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,fp8,511,0.05276266733805338
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,2047,0.19132800896962485
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,fp8,2047,0.12531200051307678
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,4095,0.3641386826833089
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,1023,0.10637332995732625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,fp8,1023,0.07563200096289317
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,fp8,4095,0.21836266915003458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,1,0.03711999952793121
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,fp8,1,0.040250666439533234
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,3,0.037920000652472176
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,8191,0.7126879692077637
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,fp8,7,0.040394666294256844
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,fp8,8191,0.40558401743570965
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,15,0.03724266588687897
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,fp8,15,0.040362666050593056
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,fp8,3,0.04068266600370407
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,31,0.03815466662247976
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,7,0.03740799923737844
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,fp8,31,0.04058666775623957
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,63,0.03920533259709676
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,127,0.045040001471837364
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,fp8,127,0.05076266825199127
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,fp8,63,0.040133332212766014
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,255,0.06894400219122569
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,fp8,255,0.0628053347269694
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,1023,0.19386667013168335
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,fp8,1023,0.13197867075602213
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,511,0.10896000266075134
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,fp8,511,0.08683733145395915
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,2047,0.3625919818878174
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,fp8,2047,0.2283573349316915
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,fp8,4095,0.413973331451416
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,4095,0.7127306461334229
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,3,0.01993600030740102
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,8191,1.5202080408732097
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,1,0.01987733319401741
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,fp8,3,0.02089066555102666
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,fp8,8191,0.7889386812845866
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,fp8,1,0.02126399924357732
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,7,0.019904000063737232
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,fp8,7,0.021114667256673176
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,15,0.019658666104078293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,31,0.020096000283956528
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,fp8,31,0.020975999534130096
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,fp8,63,0.02125866711139679
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,127,0.020725333442290623
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,fp8,127,0.023823998868465424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,255,0.027045334378878277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,63,0.019973333925008774
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,fp8,255,0.025685332715511322
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,511,0.04394133388996124
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,fp8,511,0.03271466741959254
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,1023,0.06449600060780843
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,1,0.026149332523345947
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,fp8,1023,0.04905599852403005
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,2047,0.10718933741251628
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,fp8,2047,0.07369600236415863
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,fp8,1,0.027984000742435455
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,3,0.02644266684850057
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,fp8,3,0.027823999524116516
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,7,0.025834667185942333
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,fp8,15,0.027999999622503918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,31,0.02666666607062022
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,fp8,31,0.027749332288901012
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,fp8,7,0.02775999903678894
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,15,0.026122666895389557
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,63,0.02619733413060506
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,fp8,63,0.028037334481875103
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,127,0.027808000644048054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,fp8,127,0.032672000428040825
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,255,0.046442667643229164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,fp8,15,0.02128000060717265
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,fp8,255,0.03851199895143509
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,511,0.0680320014556249
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,fp8,511,0.0555626650651296
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,1,0.03907199949026108
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,1023,0.10956266522407532
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,3,0.040192000567913055
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,fp8,1023,0.07978133360544841
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,fp8,1,0.04252799848715464
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,fp8,3,0.04228266576925913
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,7,0.038975998759269714
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,15,0.039103999733924866
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,31,0.03910933434963226
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,2047,0.19363733132680258
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,fp8,31,0.04238399863243103
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,fp8,7,0.042506664991378784
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,fp8,15,0.041893333196640015
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,63,0.03995733211437861
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,fp8,2047,0.12985066572825113
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,fp8,63,0.041893333196640015
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,127,0.04906666775544485
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,255,0.06899733344713847
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,fp8,255,0.06646933158238728
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,1023,0.19635732968648276
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,511,0.11296000083287557
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,fp8,511,0.09111467003822327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,fp8,127,0.052416001756985985
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,2047,0.36713600158691406
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,fp8,2047,0.2316640019416809
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,fp8,1023,0.13539200027783713
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,1,0.06528533498446147
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,fp8,7,0.06945600112279256
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,7,0.0652213344971339
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,3,0.06465066472689311
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,15,0.0653706689675649
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,fp8,3,0.06880000233650208
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,31,0.06749333441257477
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,fp8,31,0.06942399839560191
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,fp8,1,0.06938133140405019
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,63,0.07101333141326904
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,fp8,63,0.070783997575442
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,127,0.07561066746711731
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,fp8,127,0.09122666716575623
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,255,0.1186240017414093
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,fp8,15,0.06862399975458781
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,1023,0.36842668056488037
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,fp8,255,0.10564266641934712
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,fp8,1023,0.24406399329503378
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,2047,0.7131466865539551
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,fp8,511,0.1557866632938385
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,fp8,2047,0.4413013458251953
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,511,0.20302400986353555
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,1,0.03296533226966858
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,fp8,1,0.034272000193595886
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,3,0.033200000723203026
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,fp8,3,0.034527999659379326
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,7,0.03306666761636734
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,15,0.033301333586374916
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,fp8,31,0.03446933378775915
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,31,0.03334933271010717
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,63,0.03359466542800268
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,fp8,63,0.034741332133611046
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,127,0.035786665976047516
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,fp8,7,0.03440000116825104
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,fp8,127,0.03868799904982249
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,fp8,15,0.03453333427508672
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,255,0.05378133555253347
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,fp8,255,0.045791998505592346
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,511,0.07611200213432312
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,fp8,511,0.06276800235112508
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,1,0.04520533482233683
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,1023,0.11909332871437073
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,fp8,1,0.048725331823031105
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,fp8,1023,0.08729599912961324
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,3,0.04505600035190582
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,fp8,3,0.04726399978001913
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,7,0.044922664761543274
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,fp8,7,0.04789333542188009
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,15,0.045461331804593406
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,31,0.04552533229192098
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,63,0.04852266609668732
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,fp8,63,0.04730666677157084
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,fp8,15,0.047610665361086525
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,fp8,31,0.04747733473777771
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,127,0.0572320024172465
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,fp8,127,0.06055466830730438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,255,0.08052800099054973
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,fp8,255,0.07382933298746745
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,511,0.1204746663570404
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,fp8,511,0.09913600484530131
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,1,0.07163199782371521
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,1023,0.20552533864974976
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,fp8,1,0.07673599819342296
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,3,0.07133333384990692
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,fp8,1023,0.1449120044708252
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,fp8,3,0.07644799848397572
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,7,0.07126933336257935
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,fp8,7,0.0766133318344752
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,fp8,15,0.07674133280913036
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,31,0.07438933352629344
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,63,0.07962666451931
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,fp8,63,0.07936533292134602
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,15,0.07187733550866444
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,127,0.08523733417193095
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,fp8,127,0.09983999530474345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,fp8,31,0.07653333246707916
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,fp8,255,0.11820266644159953
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,255,0.1253600021203359
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,fp8,1,0.12959999839464822
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,1,0.12392533818880717
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,511,0.21005332469940186
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,fp8,511,0.16667733589808145
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,3,0.12353600064913432
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,fp8,3,0.13010666767756143
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,7,0.1225386659304301
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,fp8,7,0.1299626628557841
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,fp8,1023,0.2560639977455139
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,1023,0.3782879908879598
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,31,0.13032533725102743
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,fp8,31,0.13355200489362082
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,15,0.12601600090662637
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,63,0.13205333550771078
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,fp8,63,0.1376586655775706
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,fp8,15,0.1293333371480306
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,fp8,1023,0.4654346704483032
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,255,0.2158986727396647
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,1023,0.721402645111084
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,fp8,255,0.20006932814915976
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,3,0.011871999750534693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,7,0.011781333635250727
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,1,0.011482667177915573
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,7,0.012613333761692047
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,15,0.011514666179815928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,15,0.012560000022252401
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,31,0.011519999553759893
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,31,0.012602667013804117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,63,0.011445333560307821
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,63,0.012618667135636011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,127,0.011488000551859537
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,127,0.14266133308410645
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,127,0.013717333475748697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,fp8,127,0.17155200242996216
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,255,0.012725333372751871
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,fp8,511,0.2914239962895711
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,511,0.013999999811251959
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,1,0.01221866657336553
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,511,0.3912266492843628
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,1023,0.01599466676513354
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,3,0.012293333808581034
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,2047,0.02048533285657565
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,2047,0.022469334304332733
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,255,0.01394133393963178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,4095,0.032655999064445496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,8191,0.04621333380540212
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,511,0.015194666882356008
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,8191,0.05209066470464071
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,16383,0.08228266735871632
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,16383,0.0923520028591156
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,32767,0.15145066380500793
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,32767,0.17209599415461221
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,65535,0.2913973331451416
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,65535,0.32863465944925946
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,1,0.01138666644692421
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,131071,0.5643146832784017
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,1,0.012560000022252401
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,131071,0.6395839850107828
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,3,0.012426666915416718
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,3,0.012576000144084295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,7,0.011434666812419891
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,7,0.012527999778588613
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,15,0.011567999919255575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,1023,0.018181333939234417
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,15,0.01257066677014033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,4095,0.02940800040960312
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,31,0.011535999675591787
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,31,0.01259200026591619
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,63,0.011402666568756104
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,63,0.012549333274364471
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,127,0.011535999675591787
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,127,0.013925333817799887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,255,0.012837332983811697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,255,0.015743999431530636
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,511,0.014682666709025701
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,511,0.01640533283352852
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,1023,0.016762666404247284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,1023,0.018533332894245785
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,2047,0.020853333175182343
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,2047,0.023887999355793
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,4095,0.030320001145203907
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,4095,0.03323733309904734
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,16383,0.10061867038408916
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,8191,0.04948266843954722
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,8191,0.05400000015894572
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,16383,0.08433066805203755
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,32767,0.15377599994341531
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,32767,0.18397865692774454
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,65535,0.2948373357454936
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,65535,0.35097599029541016
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,1,0.01201066623131434
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,131071,0.5785866578420004
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,1,0.013151999562978745
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,3,0.011952000359694162
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,131071,0.6841973463694254
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,3,0.013194666554530462
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,7,0.011978667229413986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,7,0.013210666676362356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,31,0.013258667041858038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,15,0.012367999802033106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,15,0.013514666507641474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,31,0.012346666306257248
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,63,0.01190399999419848
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,255,0.014159999787807465
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,63,0.01340266689658165
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,127,0.012154666086037954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,127,0.0144213338692983
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,255,0.015861333658297855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,511,0.015050667027632395
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,511,0.016714667280515034
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,1023,0.016890666137139004
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,1023,0.0191040001809597
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,2047,0.021925332645575207
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,2047,0.023898666103680927
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,4095,0.04227200150489807
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,4095,0.036159999668598175
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,8191,0.06044266621271769
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,8191,0.05829333265622457
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,16383,0.1055573324362437
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,16383,0.10008000334103902
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,32767,0.20307733615239462
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,32767,0.18348799149195352
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,65535,0.38991467157999676
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,65535,0.34698665142059326
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,1,0.012133333832025528
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,1,0.01332266628742218
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,3,0.01227733368674914
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,7,0.013653332988421122
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,131071,0.7488106886545817
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,3,0.013290667285521826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,31,0.01240533341964086
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,131071,0.6861173311869303
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,7,0.012282667060693106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,15,0.012234666695197424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,15,0.013493333011865616
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,31,0.013701333353916803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,63,0.012458667159080505
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,63,0.013616000612576803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,127,0.012357333054145178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,127,0.014789332946141561
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,255,0.013973332941532135
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,255,0.016586666305859882
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,511,0.01551466683546702
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,511,0.017008000363906223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,1023,0.017818666994571686
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,1023,0.019258666783571243
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,2047,0.03642133375008901
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,2047,0.02510933329661687
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,4095,0.05826666454474131
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,4095,0.042037333051363625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,8191,0.1023573378721873
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,8191,0.06563733518123627
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,16383,0.19056532780329385
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,16383,0.11437333623568217
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,32767,0.3652373154958089
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,32767,0.20787733793258667
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,65535,0.3950666586558024
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,65535,0.7164586385091146
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,131071,0.7714506785074869
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,131071,1.533247947692871
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,fp8,1,0.03794133414824804
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,1,0.03732266773780187
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,3,0.037317333122094475
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,fp8,3,0.03821333249409994
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,7,0.0378506655494372
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,fp8,7,0.03788800040880839
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,15,0.037290667494138084
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,fp8,15,0.03787733366092046
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,31,0.03721066564321518
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,63,0.03947199881076813
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,fp8,31,0.03794133414824804
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,fp8,63,0.03786666691303253
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,127,0.0470773329337438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,fp8,127,0.04735999802748362
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,255,0.0662613312403361
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,fp8,255,0.06427200138568878
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,1,0.06373333434263866
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,fp8,511,0.0860053300857544
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,511,0.10975466171900432
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,fp8,1,0.06321066617965698
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,3,0.0641653339068095
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,fp8,3,0.062496001521746315
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,7,0.06436266501744588
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,fp8,7,0.06268266836802165
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,15,0.06462400158246358
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,31,0.06808533271153767
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,fp8,15,0.06291733185450236
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,fp8,31,0.0628053347269694
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,63,0.07149333258469899
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,fp8,63,0.06643199920654297
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,127,0.0767146646976471
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,fp8,127,0.08717333277066548
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,1,0.11494400103886922
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,255,0.11489066481590271
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,fp8,255,0.10529067118962605
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,fp8,1,0.11336533228556316
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,fp8,511,0.15424000223477682
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,511,0.20111467440923056
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,3,0.11589866876602173
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,7,0.11494933565457661
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,fp8,3,0.11244799693425496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,fp8,15,0.11314666271209717
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,fp8,7,0.11320533355077107
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,15,0.12240533034006755
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,63,0.12770133217175803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,fp8,63,0.1230560044447581
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,31,0.12584533294041952
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,fp8,31,0.11928533514340718
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,127,0.13611732920010886
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,fp8,127,0.15751999616622925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,255,0.2080693244934082
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,fp8,255,0.18744534254074097
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,1,0.2109066645304362
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,fp8,1,0.20524267355600992
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,511,0.38114134470621747
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,fp8,511,0.28109333912531537
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,3,0.2094293236732483
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,fp8,3,0.20538665850957236
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,15,0.22798933585484824
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,7,0.2212373415629069
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,fp8,15,0.21531200408935547
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,fp8,7,0.20645334323247275
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,63,0.22963200012842813
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,fp8,63,0.22216000159581503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,127,0.24671467145284018
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,fp8,127,0.28918933868408203
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,31,0.22985599438349405
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,fp8,31,0.2195840080579122
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,255,0.38273600737253827
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,fp8,255,0.3435200055440267
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,1,0.062650665640831
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,fp8,1,0.06341333190600078
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,fp8,511,0.5237386624018351
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,511,0.7393279870351156
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,fp8,3,0.0633546660343806
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,3,0.06271466612815857
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,15,0.0629013329744339
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,fp8,15,0.0632479985555013
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,fp8,7,0.06380799909432729
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,7,0.06266133487224579
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,63,0.07049066821734111
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,fp8,63,0.06648000081380208
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,127,0.07816533247629802
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,fp8,127,0.08602666854858398
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,31,0.06684799989064534
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,1,0.11576533317565918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,fp8,31,0.06340266764163971
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,fp8,1,0.11391466856002808
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,255,0.11698666214942932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,fp8,255,0.10989866654078166
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,3,0.11602133512496948
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,fp8,3,0.1132426659266154
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,7,0.11646933356920879
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,fp8,7,0.1127306620279948
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,fp8,15,0.11389866471290588
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,15,0.12346667051315308
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,31,0.12718933820724487
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,fp8,31,0.12129599849383037
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,63,0.12797866264979044
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,fp8,63,0.12576533357302347
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,fp8,127,0.1572640041510264
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,1,0.21847999095916748
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,255,0.21141332387924194
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,fp8,1,0.2133973240852356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,fp8,255,0.1927786668141683
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,3,0.2192266583442688
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,fp8,3,0.21336533625920615
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,127,0.1377226710319519
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,7,0.2312800089518229
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,fp8,7,0.21495999892552695
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,fp8,15,0.2266719937324524
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,15,0.23809067408243814
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,fp8,31,0.22961066166559854
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,31,0.2358186642328898
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,63,0.239738663037618
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,fp8,63,0.23204267024993896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,127,0.25494933128356934
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,fp8,127,0.2988800009091695
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,fp8,1,0.39743999640146893
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,1,0.4081813494364421
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,255,0.39766399065653485
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,fp8,255,0.3564480145772298
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,3,0.4331520001093547
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,fp8,3,0.3972373406092326
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,7,0.44141332308451336
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,fp8,7,0.4222559928894043
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,15,0.4416533311208089
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,fp8,15,0.4248960018157959
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,63,0.4464799960454305
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,fp8,63,0.42484267552693683
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,fp8,31,0.42425068219502765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,31,0.4411413272221883
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,1,0.01163200040658315
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,7,0.011642667154471079
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,127,0.47654934724171955
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,7,0.012378666549921036
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,3,0.0116799995303154
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,15,0.011711999773979187
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,1,0.012416000167528788
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,3,0.012250666817029318
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,31,0.011717333147923151
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,31,0.012469333906968435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,63,0.012778667112191519
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,63,0.012240000069141388
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,127,0.01181866725285848
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,fp8,127,0.5671146710713705
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,127,0.013487999637921652
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,255,0.013093333691358566
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,255,0.01444799949725469
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,511,0.015050667027632395
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,511,0.017258666455745697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,1023,0.019776000330845516
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,2047,0.027829334139823914
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,2047,0.03183999905983607
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,4095,0.04634666442871094
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,4095,0.05107733110586802
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,8191,0.08198399841785431
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,8191,0.09198400378227234
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,16383,0.15075733264287314
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,32767,0.28808534145355225
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,16383,0.16996800899505615
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,32767,0.3275573253631592
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,65535,0.5639093319574991
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,1,0.012437333663304647
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,1,0.013183999806642532
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,65535,0.6395466725031534
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,3,0.011968000481526056
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,3,0.01368533323208491
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,7,0.012005332857370377
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,7,0.01303999995191892
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,15,0.011941333611806234
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,15,0.013722666849692663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,31,0.011866666376590729
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,31,0.013077333569526672
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,63,0.012122667084137598
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,63,0.013760000467300415
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,15,0.012671999633312225
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,127,0.012026666353146235
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,127,0.014981333166360855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,255,0.013872000078360239
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,255,0.01578666642308235
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,511,0.01565333331624667
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,1023,0.021674667795499165
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,511,0.018021332720915478
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,1023,0.019541333119074505
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,1023,0.022682666778564453
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,2047,0.030069333811601002
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,2047,0.03262399882078171
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,4095,0.04789333542188009
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,4095,0.05387733379999796
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,8191,0.08442667126655579
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,8191,0.09946667154630025
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,16383,0.1543359955151876
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,16383,0.18343466520309448
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,32767,0.29337066411972046
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,255,0.7504266897837321
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,32767,0.3497920036315918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,1,0.012655999511480331
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,1,0.013679999858140945
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,65535,0.5702559947967529
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,65535,0.6866292953491211
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,3,0.012181332955757776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,7,0.01209066684047381
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,fp8,255,0.6628533204396566
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,7,0.013839999834696451
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,15,0.012613333761692047
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,15,0.013482666263977686
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,31,0.012533333152532578
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,31,0.013440000514189402
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,63,0.012383999923865
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,127,0.012565333396196365
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,127,0.015018666783968607
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,255,0.014192000031471252
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,255,0.015583999454975128
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,511,0.015706667055686314
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,511,0.018357332795858383
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,1023,0.02163200080394745
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,1023,0.022757334013779957
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,2047,0.036362667878468834
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,2047,0.033957332372665405
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,4095,0.06011199951171875
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,4095,0.05719466507434845
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,8191,0.09899733463923137
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,8191,0.10804800192515056
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,16383,0.1931040088335673
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,16383,0.18224000930786133
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,32767,0.4605226516723633
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,32767,0.34911465644836426
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,3,0.013536000003417334
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,1,0.012485332787036896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,1,0.01394133393963178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,3,0.012346666306257248
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,3,0.013760000467300415
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,7,0.01239466667175293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,65535,0.6872426668802897
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,65535,0.7300639947255453
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,7,0.013914667069911957
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,15,0.012479999413092932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,15,0.0138026662170887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,31,0.013663999736309052
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,31,0.01246400053302447
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,63,0.013861333330472311
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,63,0.012869333227475485
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,63,0.013946666071812311
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,127,0.012746666868527731
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,127,0.015146666516860327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,255,0.0145066666106383
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,255,0.01594666639963786
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,511,0.017018667111794155
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,511,0.01870399961868922
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,1023,0.03518400092919668
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,4095,0.10043733318646748
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,1023,0.02327999969323476
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,2047,0.056885331869125366
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,2047,0.04106133431196213
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,4095,0.06474666794141133
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,8191,0.1892319917678833
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,8191,0.11314666271209717
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,16383,0.36353600025177
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,16383,0.20707199970881143
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,32767,0.3953333298365275
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,1,0.011711999773979187
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,32767,0.715669314066569
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,1,0.013141332815090815
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,3,0.012266666938861212
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,3,0.013717333475748697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,65535,0.7731359799702963
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,7,0.012144000579913458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,7,0.012960000584522883
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,65535,1.530080000559489
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,15,0.012250666817029318
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,15,0.01314666618903478
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,31,0.012245333443085352
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,31,0.01328533391157786
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,63,0.013253333667914072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,63,0.011973333855470022
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,127,0.011994666109482447
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,255,0.01421333352724711
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,511,0.017845333864291508
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,127,0.014378666877746582
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,255,0.01573333392540614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,511,0.015471999843915304
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,1023,0.020207999895016353
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,1023,0.022309333086013794
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,2047,0.029834667841593426
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,2047,0.03287466615438461
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,4095,0.048725331823031105
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,8191,0.08275199929873149
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,16383,0.18212266763051352
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,8191,0.09948266545931499
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,4095,0.05240533252557119
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,16383,0.1525226632754008
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,32767,0.2918720046679179
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,32767,0.3535786469777425
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,1,0.01239466667175293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,1,0.01350933313369751
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,65535,0.5735093355178833
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,3,0.01211200033624967
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,3,0.013450667262077332
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,65535,0.6813973585764567
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,7,0.012202666451533636
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,7,0.013493333011865616
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,15,0.012400000045696894
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,15,0.013189333180586496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,31,0.012304000556468964
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,31,0.013178666432698568
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,63,0.012602667013804117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,255,0.015477333217859268
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,63,0.013573333621025085
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,127,0.012367999802033106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,127,0.015354666858911514
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,255,0.013754667093356451
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,511,0.015829333414634068
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,511,0.017781333376963932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,1023,0.02086399992307027
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,1023,0.02248000105222066
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,2047,0.0372533326347669
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,2047,0.03372266640265783
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,4095,0.05993066728115082
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,4095,0.05710400144259135
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,8191,0.1037600040435791
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,8191,0.09990933537483215
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,16383,0.20082666476567587
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,16383,0.1830400029818217
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,1,0.012624000509579977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,32767,0.3513386646906535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,1,0.01440000037352244
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,32767,0.3805066744486491
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,3,0.012757333616415659
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,3,0.014389333625634512
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,7,0.012826666235923767
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,7,0.01443733274936676
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,15,0.012629333883523941
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,15,0.014101333916187286
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,65535,0.685424009958903
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,31,0.012794667234023413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,31,0.014240000396966934
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,65535,0.7415786584218343
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,63,0.012506666282812754
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,63,0.013850666582584381
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,127,0.013194666554530462
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,255,0.014661333213249842
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,127,0.015263999501864115
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,255,0.01640533283352852
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,511,0.01878400022784869
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,511,0.0174346665541331
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,1023,0.03507733345031738
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,1023,0.02387733260790507
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,2047,0.05779199798901876
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,2047,0.04158399999141693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,4095,0.10117333134015401
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,4095,0.06498133142789204
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,8191,0.1879733403523763
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,8191,0.11506133278210957
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,1,0.011551999797423681
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,1,0.012400000045696894
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,32767,0.7139733632405599
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,16383,0.3630773226420085
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,16383,0.2074293295542399
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,3,0.011695999652147293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,32767,0.39602665106455487
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,3,0.012762666990359625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,7,0.012858666479587555
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,7,0.011711999773979187
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,15,0.012821332861979803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,65535,0.7714293003082275
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,15,0.011338666081428528
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,31,0.012479999413092932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,65535,1.524112065633138
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,31,0.01146666705608368
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,63,0.011424000064531961
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,63,0.01257066677014033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,127,0.01145600030819575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,255,0.013434667140245438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,127,0.014869333555301031
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,255,0.01458666721979777
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,511,0.014544000228246054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,511,0.01379199946920077
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,1023,0.013776000589132309
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,1023,0.014874666929244995
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,2047,0.014864000181357065
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,2047,0.01611199975013733
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,4095,0.016234666109085083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,4095,0.01699200024207433
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,8191,0.018063999712467194
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,8191,0.01960533360640208
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,16383,0.022645334402720135
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,16383,0.02516799916823705
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,32767,0.03260799994071325
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,32767,0.03484266748030981
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,65535,0.05086933573087057
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,65535,0.05578133463859558
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,131071,0.08894933263460796
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,131071,0.09872000416119893
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,1,0.011413333316644033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,1,0.012565333396196365
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,3,0.01139733319481214
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,3,0.012629333883523941
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,7,0.011546666423479715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,7,0.01232533281048139
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,15,0.012448000411192576
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,15,0.01293333371480306
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,31,0.011445333560307821
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,31,0.012778667112191519
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,63,0.011509332805871964
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,63,0.012815999488035837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,127,0.01137599969903628
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,127,0.014709333578745524
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,255,0.013487999637921652
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,255,0.014874666929244995
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,511,0.01379199946920077
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,511,0.014912000546852747
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,1023,0.01393066719174385
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,1023,0.01565333331624667
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,2047,0.015141333142916361
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,2047,0.01664000004529953
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,4095,0.016832000265518825
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,4095,0.017722666263580322
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,8191,0.019359999646743137
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,8191,0.02027200038234393
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,16383,0.023503998915354412
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,16383,0.026026666164398193
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,32767,0.03350933392842611
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,32767,0.03579733272393545
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,131071,0.10781866312026978
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,65535,0.05420800050099691
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,65535,0.05829866727193197
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,131071,0.09120532870292664
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,1,0.011578666667143503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,1,0.012453333785136541
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,15,0.011541333049535751
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,3,0.011557333171367645
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,3,0.012847999731699625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,7,0.01181866725285848
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,7,0.013007999708255133
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,15,0.012906666845083237
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,31,0.011695999652147293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,31,0.01258133351802826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,63,0.011605333536863327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,63,0.012757333616415659
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,127,0.011663999408483505
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,127,0.015135999768972397
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,255,0.013973332941532135
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,255,0.014970666418472925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,511,0.014053333550691605
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,511,0.015840000162522
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,1023,0.01462399959564209
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,1023,0.015754666179418564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,2047,0.016042667130629223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,2047,0.01735466718673706
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,4095,0.017157333592573803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,4095,0.0180479995906353
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,8191,0.019482667247454327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,8191,0.020373333245515823
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,16383,0.024405332903067272
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,16383,0.025727999707063038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,32767,0.039605334401130676
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,32767,0.038218667109807335
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,65535,0.062122667829195656
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,3,0.011461333682139715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,65535,0.06323733429114024
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,131071,0.10871466994285583
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,131071,0.10826667149861653
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,1,0.011274666835864386
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,1,0.012304000556468964
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,3,0.012437333663304647
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,7,0.011418666690587997
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,7,0.013056000073750814
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,15,0.011247999966144562
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,15,0.012794667234023413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,31,0.012618667135636011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,31,0.012757333616415659
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,63,0.01110400011142095
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,63,0.012383999923865
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,127,0.011402666568756104
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,127,0.01515199989080429
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,255,0.013541333377361298
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,255,0.01469333345691363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,511,0.013904000322024027
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,4095,0.018165333817402523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,511,0.015226667126019796
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,1023,0.014688000082969666
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,1023,0.01589866727590561
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,2047,0.01613866661985715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,2047,0.01703466723362605
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,4095,0.019461333751678467
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,8191,0.021685334543387096
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,8191,0.024154665569464367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,16383,0.03011200080315272
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,16383,0.03422933320204417
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,32767,0.04836800197760264
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,32767,0.05327466626962026
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,65535,0.0839519997437795
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,65535,0.09315733114878337
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,131071,0.15447466572125754
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,131071,0.18710933128992716
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,1,0.011626667032639185
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,1,0.01232533281048139
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,3,0.011626667032639185
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,3,0.012666666259368261
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,7,0.011578666667143503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,7,0.012618667135636011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,15,0.011711999773979187
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,15,0.012432000289360682
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,31,0.011792000383138657
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,31,0.012373333175977072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,63,0.011551999797423681
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,63,0.012837332983811697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,127,0.011648000528415045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,127,0.014607999473810196
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,255,0.013951999445756277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,255,0.014789332946141561
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,511,0.01392000044385592
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,511,0.015893333901961643
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,1023,0.015061333775520325
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,1023,0.01718933383623759
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,2047,0.01716800034046173
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,2047,0.017674667139848072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,4095,0.01926400015751521
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,4095,0.020784000555674236
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,8191,0.022543999056021374
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,8191,0.02477866659561793
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,16383,0.03207999964555105
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,16383,0.035301332672437034
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,32767,0.050111999114354454
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,32767,0.055674667159716286
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,65535,0.08802133798599243
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,65535,0.10259200135866801
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,131071,0.15822399655977884
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,131071,0.18407466014226279
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,1,0.011786667009194693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,1,0.012613333761692047
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,3,0.011754666765530905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,3,0.01321600005030632
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,7,0.011733333269755045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,31,0.01293333371480306
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,7,0.012655999511480331
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,15,0.011823999385039011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,15,0.012810666114091873
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,31,0.011727999895811081
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,63,0.011877333124478659
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,63,0.012576000144084295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,127,0.011792000383138657
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,127,0.015024000157912573
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,255,0.014111999422311783
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,255,0.01589866727590561
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,511,0.014736000448465347
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,511,0.0161920003592968
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,1023,0.016309333344300587
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,1023,0.017055999487638474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,2047,0.01738133281469345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,2047,0.017952000101407368
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,4095,0.01952533299724261
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,4095,0.02083733429511388
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,8191,0.022842665513356526
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,8191,0.02516799916823705
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,16383,0.03860799968242645
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,16383,0.03668266783157984
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,32767,0.06165866553783417
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,32767,0.06001066664854685
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,65535,0.11082133650779724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,65535,0.10148800412813823
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,131071,0.2039146622021993
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,131071,0.18593066930770874
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,1,0.013050666699806849
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,1,0.01404800017674764
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,3,0.013845333208640417
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,3,0.013104000439246496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,7,0.013088000317414602
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,7,0.014149333039919535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,15,0.01312000056107839
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,15,0.013855999956528345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,31,0.013157332936922709
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,31,0.013845333208640417
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,63,0.013072000195582708
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,63,0.013909333695967993
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,127,0.013157332936922709
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,127,0.015200000256299973
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,255,0.015135999768972397
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,255,0.015967999895413715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,511,0.016352000335852306
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,511,0.018394666413466137
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,1023,0.023242667317390442
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,1023,0.02275199939807256
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,2047,0.037205333511034645
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,2047,0.03455466777086258
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,8191,0.10846400260925293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,8191,0.10130666693051656
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,4095,0.060138667623202004
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,4095,0.058821335434913635
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,16383,0.18593599398930868
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,1,0.012847999731699625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,16383,0.19432000319163004
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,1,0.014271999398867289
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,3,0.02426133304834366
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,7,0.013466666142145792
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,3,0.01458666721979777
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,7,0.013936000565687815
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,15,0.014069333672523499
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,15,0.012863999853531519
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,32767,0.3527146577835083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,32767,0.4682933489481608
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,31,0.014090667168299357
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,31,0.013418667018413544
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,63,0.01328533391157786
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,63,0.014074667046467463
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,127,0.013440000514189402
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,127,0.015578666081031164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,255,0.015285332997639975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,255,0.016154666741689045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,511,0.017157333592573803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,511,0.018976000448067982
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,1023,0.03641066700220108
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,1023,0.024282666544119518
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,2047,0.05762666463851929
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,2047,0.04167466859022776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,4095,0.1011946698029836
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,4095,0.06733866532643636
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,8191,0.18901866674423218
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,8191,0.11734400192896526
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,1,0.01732800031701724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,1,0.018624000251293182
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,3,0.017632000148296356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,16383,0.364464004834493
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,16383,0.20888533194859824
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,3,0.01838933303952217
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,7,0.017157333592573803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,7,0.01828266680240631
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,32767,0.4002986749013265
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,32767,0.7141866683959961
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,15,0.017008000363906223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,15,0.017935999979575474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,31,0.018191999445358913
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,31,0.017136000096797943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,63,0.017525333911180496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,127,0.021344001094500225
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,63,0.018538666268189747
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,255,0.022672000030676525
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,255,0.021104000508785248
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,127,0.017445333302021027
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,511,0.029274667302767437
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,511,0.03878933439652125
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,2047,0.10237866640090942
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,1023,0.05939733485380808
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,2047,0.07009600102901459
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,1023,0.0455626646677653
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,4095,0.18941867351531982
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,4095,0.11684800187746684
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,8191,0.21235199769337973
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,1,0.011247999966144562
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,1,0.012298667182525
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,3,0.011178666104873022
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,8191,0.3617333173751831
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,3,0.012506666282812754
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,7,0.011226666470368704
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,16383,0.3972426652908325
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,7,0.012298667182525
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,15,0.011039999624093374
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,32767,0.7752160231272379
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,16383,0.7114346822102865
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,15,0.012517333030700684
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,31,0.012432000289360682
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,63,0.010762666662534079
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,31,0.011157333850860596
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,255,0.013280000537633896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,127,0.014592000593741735
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,63,0.011866666376590729
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,32767,1.5236053466796875
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,255,0.01482133318980535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,127,0.010816000401973724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,511,0.014479999740918478
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,511,0.015909332782030106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,1023,0.01568000018596649
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,1023,0.016757333030303318
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,2047,0.01695999999841054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,4095,0.021338666478792827
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,4095,0.023562667270501454
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,16383,0.053082664807637535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,2047,0.019343999524911244
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,8191,0.029850666721661884
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,8191,0.033610666791598
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,16383,0.048714667558670044
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,32767,0.08483733733495076
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,32767,0.09313066800435384
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,65535,0.1529759963353475
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,65535,0.18570133050282797
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,131071,0.2897866765658061
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,1,0.011168000598748526
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,131071,0.35257065296173096
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,1,0.012149333953857422
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,3,0.011168000598748526
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,3,0.01201066623131434
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,7,0.01118933285276095
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,7,0.012426666915416718
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,15,0.011120000233252844
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,15,0.012186666329701742
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,31,0.011365332951148352
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,31,0.0122079998254776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,63,0.011050666371981302
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,63,0.012144000579913458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,127,0.011264000087976456
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,127,0.014581333845853806
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,255,0.01360000049074491
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,255,0.015541333705186844
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,511,0.015040000279744467
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,511,0.01687466725707054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,4095,0.022634667654832203
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,1023,0.016496000190575916
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,1023,0.017162666966517765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,2047,0.017551999539136887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,2047,0.020069333414236706
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,4095,0.024400000770886738
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,8191,0.030746666093667347
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,32767,0.10136533776919048
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,8191,0.03435199956099192
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,16383,0.05134400228659312
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,16383,0.05514133473237356
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,32767,0.08437333504358928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,65535,0.15850133697191873
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,65535,0.184005339940389
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,131071,0.29773332675298053
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,1,0.011770666887362799
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,131071,0.3526666561762492
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,1,0.012400000045696894
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,3,0.011605333536863327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,3,0.012645332763592402
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,7,0.01180800050497055
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,7,0.012421333541472753
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,15,0.011749333391586939
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,15,0.012522666404644648
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,31,0.011498666057984034
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,31,0.012527999778588613
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,63,0.011898666620254517
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,63,0.012469333906968435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,127,0.011706666400035223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,127,0.016021333634853363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,255,0.014373333503802618
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,255,0.015429332852363586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,511,0.015930666277805965
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,511,0.017509333789348602
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,1023,0.01722666621208191
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,1023,0.01754666616519292
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,2047,0.018277333428462345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,2047,0.02019199977318446
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,4095,0.023685333629449207
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,4095,0.024746666351954143
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,8191,0.03822399924198786
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,8191,0.03676799933115641
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,32767,0.1018453339735667
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,16383,0.07237866520881653
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,65535,0.2416426738103231
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,65535,0.18349866072336832
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,16383,0.06011199951171875
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,131071,0.38124267260233563
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,32767,0.10621333122253418
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,131071,0.3500320116678874
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,1,0.01414399966597557
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,3,0.014325333138306936
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,1,0.015072000523408255
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,3,0.015354666858911514
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,7,0.014730667074521383
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,7,0.01545599972208341
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,15,0.014709333578745524
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,15,0.01552533358335495
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,31,0.014298666268587112
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,31,0.015274666249752045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,63,0.014064000298579534
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,63,0.01545599972208341
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,127,0.01481066644191742
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,127,0.01693333312869072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,255,0.016208000481128693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,255,0.01785600061217944
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,511,0.0183146670460701
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,511,0.020469332734743755
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,1023,0.037231999138991036
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,1023,0.025461333493391674
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,2047,0.05850133299827576
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,2047,0.04390400151411692
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,8191,0.19005866845448813
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,8191,0.1178986628850301
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,4095,0.10426666339238484
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,4095,0.0693280001481374
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,1,0.018021332720915478
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,16383,0.3656533161799113
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,3,0.01759999990463257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,16383,0.21310933430989584
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,1,0.01915733392039935
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,15,0.018138666947682697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,3,0.019386666516462963
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,15,0.018965333700180054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,7,0.018906666586796444
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,31,0.017887999614079792
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,63,0.017802666872739792
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,7,0.01802666609485944
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,31,0.01904533306757609
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,63,0.01931200052301089
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,127,0.021754667162895203
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,127,0.018613333503405254
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,255,0.022218666970729828
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,255,0.023418667415777843
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,511,0.03963200002908707
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,511,0.02958933264017105
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,1023,0.0610453337430954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,1023,0.04706666866938273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,4095,0.19109867016474405
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,4095,0.12036800384521484
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,2047,0.10410666465759277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,8191,0.36448001861572266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,2047,0.07297066847483318
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,1,0.024421334266662598
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,1,0.02643200010061264
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,8191,0.21355199813842773
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,3,0.026394667724768322
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,3,0.024346667031447094
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,7,0.026159999271233875
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,15,0.024506665766239166
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,16383,0.4018346468607585
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,7,0.024117333193620045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,16383,0.7118879954020182
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,31,0.02437866727511088
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,15,0.026421333352724712
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,31,0.02628266563018163
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,63,0.02476799984773
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,127,0.025600001215934753
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,255,0.04338666796684265
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,255,0.03678400069475174
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,63,0.02682666728893916
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,511,0.06482133269309998
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,511,0.053413331508636475
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,127,0.0308693324526151
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,2047,0.19058666626612344
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,2047,0.12460266550381978
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,1023,0.10556800166765849
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,1023,0.07632533212502797
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,8191,0.4062986771265666
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,4095,0.3630666732788086
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,4095,0.21757866938908896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,8191,0.712224006652832
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,1,0.019850666324297588
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,fp8,1,0.021573332448800404
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,3,0.020197333147128422
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,fp8,3,0.02111999938885371
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,16383,0.7776906490325928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,fp8,7,0.02107200026512146
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,16383,1.522821267445882
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,7,0.019978666057189304
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,15,0.01966399947802226
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,fp8,15,0.02125866711139679
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,31,0.019727999965349834
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,fp8,31,0.021066665649414062
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,fp8,63,0.02124800036350886
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,fp8,127,0.024112001061439514
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,255,0.02402666707833608
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,fp8,255,0.025914666553338368
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,63,0.019733333339293797
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,511,0.04171200096607208
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,fp8,511,0.03216533362865448
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,1023,0.06451199948787689
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,fp8,1023,0.05013866722583771
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,1,0.025909334421157837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,127,0.02033599962790807
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,2047,0.10748266180356343
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,fp8,2047,0.07521066566308339
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,fp8,1,0.02771199991305669
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,3,0.025936000049114227
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,fp8,3,0.027802666028340656
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,fp8,4095,0.1223360002040863
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,7,0.025957333544890087
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,4095,0.19500267505645752
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,fp8,15,0.02790933350721995
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,fp8,7,0.02773333340883255
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,31,0.026330667237440746
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,15,0.026250667870044708
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,fp8,31,0.027600000301996868
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,fp8,63,0.028069332242012024
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,fp8,127,0.03244800120592117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,255,0.045738667249679565
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,fp8,255,0.03757333258787791
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,511,0.06777599950631459
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,fp8,511,0.05477866530418396
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,1023,0.1083733340104421
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,63,0.026496000587940216
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,fp8,1023,0.07885333398977916
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,127,0.026687999566396076
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,fp8,4095,0.22073600689570108
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,4095,0.36663464705149335
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,fp8,1,0.04214400053024292
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,1,0.039034667114416756
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,fp8,2047,0.12812800208727518
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,2047,0.19309866428375244
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,3,0.03882133215665817
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,fp8,3,0.0418453315893809
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,7,0.03886933376391729
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,fp8,7,0.04232533276081085
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,15,0.039247999588648476
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,fp8,15,0.04250133534272512
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,63,0.040847999354203544
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,fp8,63,0.04218133290608724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,127,0.04756799836953481
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,fp8,127,0.053130666414896645
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,255,0.0713973343372345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,fp8,255,0.0653546651204427
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,fp8,511,0.08975999553998311
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,511,0.11177600423494975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,31,0.038933334251244865
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,fp8,31,0.041936000188191734
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,2047,0.3652640183766683
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,fp8,2047,0.23225067059199014
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,fp8,1023,0.13593066732088724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,1023,0.19527467091878256
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,4095,0.7121013005574545
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,fp8,4095,0.41741331418355304
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,1,0.0314026673634847
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,fp8,1,0.032885332902272545
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,3,0.03178133318821589
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,fp8,3,0.0334346666932106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,15,0.03146666785081228
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,fp8,15,0.03324799984693527
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,31,0.03145600110292435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,fp8,31,0.032933334509531655
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,7,0.031130666534105938
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,63,0.03178666780392329
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,fp8,7,0.03297066688537598
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,fp8,63,0.0330826664964358
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,fp8,127,0.03714666763941447
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,127,0.033904001116752625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,255,0.05179733534653982
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,fp8,255,0.04526400069395701
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,511,0.073253333568573
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,fp8,511,0.06132799883683523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,1023,0.114138662815094
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,1,0.0428959975639979
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,fp8,1023,0.08543999989827473
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,fp8,1,0.0461706668138504
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,3,0.042805333932240806
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,fp8,2047,0.13481600085894266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,fp8,3,0.04539200166861216
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,2047,0.20060267051060995
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,7,0.042837331692377724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,fp8,7,0.04577599962552389
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,fp8,15,0.04635733366012573
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,31,0.04322666426499685
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,15,0.04297066728274027
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,63,0.0452159990866979
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,127,0.05452266832192739
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,fp8,127,0.05699199934800466
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,255,0.07562666634718578
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,fp8,255,0.07131733496983846
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,fp8,31,0.0455626646677653
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,511,0.11885866522789001
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,fp8,511,0.09598400195439656
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,fp8,63,0.046629334489504494
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,1,0.06931200126806895
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,1023,0.20220800240834555
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,fp8,1,0.07448533177375793
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,7,0.06925333539644878
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,fp8,1023,0.14281599720319113
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,3,0.06914666791756947
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,fp8,3,0.07473599910736084
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,15,0.06921066840489705
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,fp8,2047,0.23894399404525757
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,fp8,7,0.07473066449165344
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,fp8,15,0.07464000085989635
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,2047,0.372874657313029
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,fp8,31,0.07425066828727722
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,fp8,63,0.07789866626262665
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,63,0.07593066493670146
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,fp8,127,0.0972213347752889
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,255,0.12178666392962138
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,511,0.20625599225362143
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,31,0.07077333331108093
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,1,0.011690666278203329
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,1,0.012495999534924826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,3,0.011616000284751257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,3,0.0124746672809124
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,7,0.011594666788975397
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,fp8,511,0.1643946667512258
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,7,0.012421333541472753
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,15,0.012586666891972223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,31,0.011535999675591787
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,2047,0.7204586664835612
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,31,0.012661332885424295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,63,0.012586666891972223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,fp8,255,0.11472533146540324
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,127,0.08309866487979889
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,127,0.011530666301647821
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,63,0.011274666835864386
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,fp8,1023,0.25326399008433026
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,255,0.01565333331624667
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,511,0.014869333555301031
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,1023,0.017360000560681026
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,fp8,2047,0.44781335194905597
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,1023,0.019018666197856266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,255,0.012543999900420507
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,15,0.011370666325092316
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,1023,0.37379733721415204
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,2047,0.021375998854637146
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,4095,0.033941333492596946
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,127,0.014298666268587112
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,4095,0.02997333308060964
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,8191,0.04855466882387797
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,2047,0.023893333971500397
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,16383,0.08454400300979614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,16383,0.09778666496276855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,32767,0.1534293293952942
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,32767,0.18561599651972452
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,65535,0.29078932603200275
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,65535,0.3528746763865153
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,1,0.011695999652147293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,131071,0.6878026326497396
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,3,0.011493333925803503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,3,0.0124746672809124
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,131071,0.5697653293609619
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,1,0.0129120002190272
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,7,0.01146666705608368
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,7,0.012560000022252401
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,511,0.016575999557971954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,15,0.011503999431928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,15,0.012794667234023413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,31,0.011690666278203329
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,8191,0.05276800195376078
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,31,0.012863999853531519
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,63,0.011525332927703857
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,63,0.012453333785136541
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,127,0.011541333049535751
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,511,0.016293333222468693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,127,0.014149333039919535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,255,0.013077333569526672
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,255,0.01672533278663953
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,511,0.017322666943073273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,1023,0.0174346665541331
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,1023,0.019637333850065868
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,2047,0.022309333086013794
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,2047,0.024746666351954143
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,4095,0.03204799940188726
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,4095,0.03431999931732813
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,8191,0.05048533280690511
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,8191,0.05513600011666616
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,16383,0.08549333612124126
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,16383,0.10146666566530864
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,32767,0.1564906636873881
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,32767,0.1844480037689209
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,65535,0.29236799478530884
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,65535,0.3513813416163127
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,1,0.012159999459981918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,1,0.013397333522637686
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,131071,0.5745813449223837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,3,0.012042666474978128
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,131071,0.683631976445516
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,3,0.013354666531085968
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,7,0.011909333368142446
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,7,0.013610667238632837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,15,0.012047999848922094
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,15,0.013125333935022354
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,63,0.01328533391157786
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,127,0.012122667084137598
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,31,0.012144000579913458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,31,0.013104000439246496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,63,0.01209066684047381
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,127,0.014661333213249842
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,255,0.013712000101804733
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,1023,0.019578666736682255
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,255,0.01708799973130226
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,511,0.01634666696190834
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,511,0.01747200017174085
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,1023,0.018677332748969395
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,2047,0.02390933285156886
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,2047,0.02518400053183238
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,4095,0.043375998735427856
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,16383,0.10186666250228882
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,4095,0.03745600084463755
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,8191,0.07173333565394084
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,32767,0.18585066000620523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,8191,0.05917333563168844
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,16383,0.12869866689046225
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,32767,0.20269866784413657
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,65535,0.4642293453216553
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,65535,0.3516053358713786
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,131071,0.9180746873219808
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,131071,0.681552012761434
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,1,0.038634667793909706
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,fp8,1,0.038848000268141426
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,3,0.038373333712418876
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,7,0.03817066550254822
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,fp8,7,0.03890133400758108
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,fp8,3,0.039290666580200195
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,fp8,15,0.03865066667397817
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,15,0.03827733298142751
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,31,0.03853866706291834
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,fp8,31,0.039034667114416756
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,63,0.04038399954636892
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,fp8,63,0.03903999924659729
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,127,0.046767999728520714
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,255,0.06573866804440816
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,fp8,127,0.04776533444722494
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,fp8,255,0.06387733419736226
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,511,0.1113920013109843
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,fp8,511,0.08744000395139058
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,1,0.06459733347098033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,fp8,1,0.0626933326323827
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,1023,0.1948853333791097
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,fp8,1023,0.1341493328412374
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,3,0.06397866706053416
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,fp8,3,0.06302399933338165
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,15,0.06444799900054932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,7,0.0643039991458257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,fp8,15,0.06266133487224579
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,fp8,7,0.06313600142796834
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,fp8,63,0.06513066589832306
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,63,0.07097599903742473
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,127,0.0757013310988744
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,fp8,127,0.08717866738637288
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,31,0.06823466718196869
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,fp8,31,0.06281066437562306
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,255,0.11285866300264995
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,fp8,255,0.10474133491516113
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,fp8,511,0.15309866269429526
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,511,0.19892799854278564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,1,0.11515733599662781
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,fp8,1,0.11297600467999776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,1023,0.3691946665445964
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,3,0.11570133765538533
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,fp8,1023,0.2460106611251831
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,fp8,7,0.11310399572054546
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,15,0.11979200442632039
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,7,0.11506666739781697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,fp8,3,0.11238933602968852
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,31,0.12569600343704224
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,63,0.1272640029589335
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,fp8,63,0.12409067153930664
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,127,0.13540800412495932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,fp8,127,0.15769066413243613
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,fp8,15,0.11319466431935628
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,fp8,255,0.1869600017865499
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,255,0.20480000972747803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,fp8,31,0.11822932958602905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,511,0.379802664120992
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,fp8,511,0.2828213373819987
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,1,0.06489066779613495
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,fp8,1,0.065610667069753
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,1023,0.7624586423238119
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,3,0.06464533507823944
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,fp8,3,0.06588799754778545
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,fp8,1023,0.4541066487630208
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,7,0.06499200065930684
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,15,0.06512533128261566
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,fp8,7,0.06553600231806438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,fp8,31,0.06558933357397716
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,63,0.07291733225186665
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,31,0.0690880020459493
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,127,0.0795306662718455
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,fp8,127,0.08885332942008972
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,fp8,15,0.06594133377075195
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,255,0.11544000109036763
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,fp8,255,0.11158933242162068
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,511,0.2006453275680542
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,1,0.116047998269399
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,fp8,1,0.11306666334470113
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,3,0.11661332845687866
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,fp8,63,0.0708000014225642
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,fp8,3,0.11320533355077107
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,7,0.11506133278210957
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,fp8,7,0.11331733067830403
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,fp8,511,0.15914133191108704
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,15,0.12237866719563802
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,fp8,15,0.11322666207949321
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,31,0.12664533654848734
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,fp8,31,0.11927466591199239
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,63,0.12738133470217386
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,fp8,63,0.12358933687210083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,127,0.13710932930310568
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,255,0.20837867259979248
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,fp8,127,0.15709333618481955
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,fp8,255,0.19223999977111816
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,1,0.21799467007319132
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,511,0.3815466562906901
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,fp8,511,0.2885013421376546
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,fp8,1,0.21149865786234537
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,3,0.2178666591644287
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,7,0.22719999154408774
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,fp8,3,0.2130933403968811
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,fp8,7,0.2132479945818583
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,15,0.2357706626256307
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,fp8,15,0.22331732511520386
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,fp8,31,0.2288693388303121
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,31,0.23573333024978638
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,63,0.23851199944814047
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,fp8,63,0.23119467496871948
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,127,0.25328532854715985
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,fp8,127,0.29874666531880695
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,1,0.011642667154471079
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,1,0.012629333883523941
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,3,0.012479999413092932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,7,0.012725333372751871
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,15,0.011503999431928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,15,0.012869333227475485
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,31,0.011296000331640244
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,31,0.012682666381200155
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,63,0.011503999431928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,63,0.012815999488035837
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,127,0.011519999553759893
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,127,0.013925333817799887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,255,0.01369599997997284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,255,0.014901333798964819
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,255,0.3927253484725952
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,511,0.014853333433469137
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,511,0.017514667163292568
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,1023,0.019280000279347103
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,2047,0.02787200113137563
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,3,0.011551999797423681
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,2047,0.03181333343187968
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,4095,0.046256000796953835
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,7,0.01139733319481214
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,4095,0.05095999936262766
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,8191,0.08191999793052673
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,8191,0.09341866771380107
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,16383,0.18395199378331503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,16383,0.1511360009511312
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,32767,0.28937600056330365
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,32767,0.35100265343983966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,65535,0.5663040081659952
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,65535,0.6839413642883301
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,1,0.01227733368674914
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,1,0.0138026662170887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,fp8,255,0.35688531398773193
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,131071,1.1204053560892742
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,3,0.012773333738247553
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,1023,0.022181332111358643
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,7,0.013466666142145792
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,131071,1.345344066619873
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,3,0.013370666652917862
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,15,0.01179733375708262
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,15,0.013466666142145792
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,31,0.011946666985750198
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,31,0.013306666165590286
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,63,0.012015999605258306
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,63,0.013397333522637686
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,127,0.0120319997270902
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,127,0.014949332922697067
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,255,0.013733333597580591
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,255,0.015754666179418564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,511,0.015466666469971338
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,511,0.018144000321626663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,1023,0.019567999988794327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,1023,0.023002666731675465
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,7,0.01232533281048139
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,2047,0.028757333755493164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,2047,0.03270933280388514
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,4095,0.04929066697756449
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,4095,0.052671998739242554
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,8191,0.08334933718045552
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,16383,0.15498133500417074
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,8191,0.09987200299898784
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,16383,0.1823306679725647
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,32767,0.29179733991622925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,32767,0.3531680107116699
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,65535,0.5764106512069702
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,1,0.012527999778588613
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,65535,0.6839306354522705
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,1,0.014005333185195923
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,3,0.0129120002190272
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,3,0.013855999956528345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,131071,1.1322933038075764
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,7,0.012416000167528788
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,7,0.013647999614477158
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,15,0.013631999492645264
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,131071,1.3406933148701985
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,15,0.01231466606259346
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,31,0.013386666774749756
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,31,0.012682666381200155
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,63,0.012768000364303589
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,63,0.01379199946920077
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,255,0.01423466702302297
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,127,0.012624000509579977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,127,0.015024000157912573
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,255,0.016165333489576977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,511,0.7429920037587484
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,1023,0.022991999983787537
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,511,0.015775999675194424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,511,0.018165333817402523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,fp8,511,0.5441013177235922
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,1023,0.02107200026512146
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,2047,0.036101333796978
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,2047,0.03562666724125544
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,8191,0.09924800197283427
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,4095,0.05929600199063619
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,4095,0.05779199798901876
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,16383,0.2014133334159851
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,16383,0.18416533867518106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,32767,0.3498293161392212
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,65535,0.7647093137105306
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,1,0.012768000364303589
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,65535,0.6820480028788248
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,1,0.013199999928474426
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,3,0.01211200033624967
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,3,0.013114667187134424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,131071,1.3419359525044758
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,131071,1.5887519518534343
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,7,0.01312000056107839
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,7,0.011887999872366587
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,15,0.011839999506870905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,15,0.013077333569526672
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,63,0.012191999703645706
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,31,0.012202666451533636
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,63,0.013056000073750814
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,127,0.012144000579913458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,31,0.01312000056107839
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,127,0.015344000111023584
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,255,0.013418667018413544
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,255,0.014954666296641031
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,511,0.015072000523408255
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,511,0.017653333644072216
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,1023,0.022485333184401195
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,1023,0.019653332730134327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,2047,0.029711998999118805
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,2047,0.032127998769283295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,4095,0.04890666902065277
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,4095,0.05454933146635691
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,8191,0.08357333143552144
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,8191,0.10010666648546855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,16383,0.15286399920781454
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,16383,0.1839946707089742
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,32767,0.35040533542633057
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,32767,0.29425599177678424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,65535,0.6848320166269938
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,1,0.012576000144084295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,65535,0.5736639897028605
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,1,0.01321600005030632
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,3,0.012191999703645706
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,7,0.011909333368142446
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,8191,0.1081813375155131
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,3,0.013834666460752487
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,131071,1.1260746320088704
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,7,0.013306666165590286
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,131071,1.344922701517741
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,15,0.012063999970753988
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,31,0.012389333297808966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,63,0.012144000579913458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,32767,0.38807467619578045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,15,0.013525333255529404
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,31,0.013418667018413544
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,127,0.012282667060693106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,63,0.013194666554530462
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,127,0.015376000354687372
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,255,0.014352000008026758
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,255,0.015285332997639975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,511,0.015770666301250458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,511,0.018229333062966663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,1023,0.02146133283774058
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,1023,0.023039999107519787
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,2047,0.03551466763019562
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,2047,0.04101333270470301
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,4095,0.060549333691596985
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,4095,0.05747200051943461
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,8191,0.10820266604423523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,8191,0.10012267033259074
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,16383,0.2013333241144816
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,16383,0.18334933121999106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,1,0.011312000453472137
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,32767,0.49034667015075684
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,1,0.012367999802033106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,65535,0.7397493521372477
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,32767,0.34991466999053955
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,3,0.012901333471139273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,3,0.012896000097195307
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,65535,0.6864799658457438
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,7,0.012778667112191519
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,15,0.01228800043463707
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,7,0.012442667037248611
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,15,0.01138666644692421
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,31,0.011551999797423681
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,131071,1.3447093963623047
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,63,0.011296000331640244
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,31,0.012426666915416718
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,255,0.013797332843144735
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,131071,1.5726186434427898
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,63,0.012229333321253458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,127,0.011488000551859537
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,127,0.015253332753976187
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,511,0.013925333817799887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,255,0.015168000012636185
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,511,0.014730667074521383
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,1023,0.014111999422311783
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,2047,0.014997333288192749
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,1023,0.015024000157912573
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,2047,0.01825599993268649
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,4095,0.01618133361140887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,4095,0.017136000096797943
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,8191,0.018320000420014065
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,8191,0.019706666469573975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,16383,0.022976001103719074
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,16383,0.02513066679239273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,32767,0.03221333275238673
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,32767,0.03536533315976461
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,65535,0.05085866649945577
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,65535,0.05622399846712748
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,131071,0.0890933374563853
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,131071,0.09913067022959392
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,1,0.01126933346192042
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,1,0.012421333541472753
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,3,0.01146666705608368
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,3,0.012746666868527731
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,7,0.011242666592200598
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,7,0.012400000045696894
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,15,0.011434666812419891
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,15,0.0124746672809124
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,31,0.011434666812419891
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,31,0.012330666184425354
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,63,0.011370666325092316
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,63,0.012768000364303589
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,127,0.011312000453472137
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,127,0.01481066644191742
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,255,0.013882666826248169
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,255,0.015008000036080679
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,511,0.013967999567588171
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,511,0.01481066644191742
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,1023,0.014186666657527288
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,1023,0.015728000551462173
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,2047,0.015226667126019796
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,2047,0.016885332763195038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,4095,0.016917333006858826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,4095,0.017840000490347546
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,8191,0.01912533367673556
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,8191,0.0205226664741834
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,16383,0.0234400009115537
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,16383,0.025813333690166473
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,32767,0.033904001116752625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,32767,0.03603200117746989
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,65535,0.05459733307361603
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,65535,0.058186665177345276
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,131071,0.09030933181444804
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,1,0.011114666859308878
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,131071,0.10820266604423523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,1,0.01251199965675672
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,3,0.011413333316644033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,3,0.012768000364303589
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,7,0.01126933346192042
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,7,0.012837332983811697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,15,0.011231999844312668
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,15,0.012869333227475485
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,31,0.011546666423479715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,31,0.012485332787036896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,63,0.01139733319481214
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,63,0.01239466667175293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,127,0.01138666644692421
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,127,0.014789332946141561
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,255,0.014192000031471252
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,255,0.014991999914248785
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,511,0.014138666292031607
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,511,0.015279999623696009
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,1023,0.014970666418472925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,1023,0.016069332758585613
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,2047,0.016250666230916977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,2047,0.017344000438849132
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,4095,0.018650667121013004
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,4095,0.01974933346112569
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,8191,0.02161066730817159
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,8191,0.024405332903067272
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,16383,0.03046933313210805
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,16383,0.03437866767247518
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,32767,0.048709332942962646
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,32767,0.05349866549173991
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,65535,0.08444266517957051
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,65535,0.09735999504725139
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,131071,0.15366400281588236
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,131071,0.1848906675974528
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,1,0.011498666057984034
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,1,0.012698666503032049
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,3,0.012831999609867731
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,3,0.011535999675591787
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,7,0.011530666301647821
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,7,0.013002666334311167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,15,0.011312000453472137
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,15,0.012826666235923767
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,31,0.011381333072980246
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,31,0.012800000607967377
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,63,0.01163200040658315
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,63,0.012586666891972223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,127,0.01157333329319954
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,127,0.01525866612792015
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,255,0.01404800017674764
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,255,0.015392000476519266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,511,0.014293332894643148
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,511,0.01579733317097028
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,1023,0.015392000476519266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,1023,0.017269333203633625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,2047,0.01728533332546552
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,2047,0.017759999881188076
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,4095,0.01926400015751521
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,16383,0.035504000882307686
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,4095,0.0206133338312308
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,8191,0.022463999688625336
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,8191,0.02517866591612498
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,16383,0.03164266546567281
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,32767,0.05065066615740458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,32767,0.05715733269850413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,65535,0.08562666177749634
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,65535,0.10292800267537434
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,131071,0.1546453336874644
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,131071,0.18473599354426065
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,1,0.013237333546082178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,1,0.013647999614477158
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,3,0.013056000073750814
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,3,0.014058666924635569
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,7,0.01293333371480306
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,7,0.013647999614477158
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,15,0.013061333447694778
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,15,0.013642666240533194
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,31,0.013034666577974955
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,31,0.01379199946920077
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,63,0.013093333691358566
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,63,0.01413333291808764
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,127,0.01313599944114685
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,127,0.015200000256299973
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,255,0.014458666245142618
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,255,0.016042667130629223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,511,0.016677333662907284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,511,0.01826133330663045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,1023,0.022837333381175995
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,1023,0.020960000654061634
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,2047,0.03714666763941447
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,2047,0.03510933369398117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,4095,0.059818665186564125
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,8191,0.10656000177065532
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,4095,0.058693334460258484
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,8191,0.10080533226331075
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,16383,0.23974400758743286
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,16383,0.1851200064023336
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,32767,0.395466685295105
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,1,0.013370666652917862
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,1,0.014170666535695394
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,3,0.01332266628742218
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,32767,0.35307733217875165
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,65535,0.769050677617391
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,7,0.01321600005030632
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,3,0.014181333283583323
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,7,0.01393066719174385
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,15,0.013466666142145792
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,65535,0.6873813470204672
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,15,0.014015999933083853
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,31,0.013973332941532135
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,63,0.01332266628742218
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,63,0.014159999787807465
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,31,0.01340266689658165
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,127,0.013690666606028875
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,255,0.015520000209410986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,255,0.01616000011563301
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,127,0.015498666713635126
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,511,0.018090666582187016
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,511,0.01869333287080129
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,1023,0.03612266729275385
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,1023,0.023946667710940044
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,2047,0.05772800246874491
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,2047,0.04155199974775314
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,4095,0.06653866668542226
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,4095,0.10205333431561787
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,8191,0.1893706719080607
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,8191,0.1162559986114502
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,16383,0.3640586535135905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,1,0.011109333485364914
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,16383,0.21066667636235556
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,3,0.010863999525705973
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,3,0.011711999773979187
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,1,0.012661332885424295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,7,0.012213333199421564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,15,0.010768000036478043
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,15,0.013002666334311167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,7,0.011066666493813196
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,31,0.011749333391586939
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,31,0.011829332758982977
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,32767,0.39589866002400714
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,32767,0.714576005935669
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,63,0.011381333072980246
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,63,0.012026666353146235
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,65535,0.7852053642272949
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,127,0.011050666371981302
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,511,0.014943999548753103
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,65535,1.5232586860656738
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,127,0.01504533365368843
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,255,0.01351999988158544
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,511,0.01587733378012975
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,255,0.01509333277742068
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,1023,0.016901332885026932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,2047,0.0173333336909612
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,1023,0.01617066686352094
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,4095,0.021551998953024547
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,4095,0.024293333292007446
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,8191,0.030042665700117748
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,2047,0.01953599974513054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,8191,0.034048000971476235
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,16383,0.04863466819127401
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,16383,0.05356800059477488
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,32767,0.0846560001373291
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,32767,0.09682133793830872
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,65535,0.15427199999491373
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,65535,0.18554667631785074
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,131071,0.2892000079154968
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,1,0.01121066634853681
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,131071,0.3531839847564697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,1,0.01267733300725619
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,3,0.011429333438475927
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,3,0.012186666329701742
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,7,0.011157333850860596
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,7,0.012080000092585882
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,15,0.011146667102972666
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,15,0.012357333054145178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,31,0.011130666981140772
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,31,0.013072000195582708
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,63,0.011461333682139715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,63,0.012175999581813812
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,511,0.017018667111794155
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,127,0.011226666470368704
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,1023,0.01684800038735072
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,127,0.014869333555301031
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,255,0.01394133393963178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,255,0.015824000040690105
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,511,0.015205333630243937
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,1023,0.01752000053723653
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,2047,0.01846933364868164
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,2047,0.02027200038234393
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,4095,0.02237333357334137
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,4095,0.024618667860825855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,8191,0.0317546675602595
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,8191,0.034714666505654655
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,32767,0.10216533144315083
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,16383,0.05165866514046987
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,16383,0.05666666726271311
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,32767,0.0862559974193573
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,65535,0.1543839971224467
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,65535,0.1851253310839335
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,131071,0.2982826630274455
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,131071,0.3529386520385742
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,1,0.014592000593741735
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,1,0.015461333096027374
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,3,0.014511999984582266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,3,0.015477333217859268
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,7,0.014469332993030548
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,7,0.015360000232855478
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,15,0.014458666245142618
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,15,0.015290666371583939
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,31,0.01452800010641416
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,31,0.015413332730531693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,63,0.014538666854302088
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,63,0.01575999955336253
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,127,0.014362666755914688
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,127,0.017077332983414333
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,255,0.016095999628305435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,255,0.017808000246683758
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,511,0.01850133389234543
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,511,0.020479999482631683
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,1023,0.036992001036802925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,1023,0.025114665428797405
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,2047,0.059301331639289856
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,2047,0.04412800073623657
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,8191,0.19130132595698038
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,8191,0.11773332953453064
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,4095,0.10318400462468465
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,4095,0.0687306672334671
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,16383,0.36634135246276855
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,1,0.018239999810854595
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,1,0.018976000448067982
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,3,0.0180479995906353
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,7,0.019941333681344986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,7,0.01817600056529045
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,3,0.019541333119074505
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,15,0.01834133391578992
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,15,0.0195573332409064
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,31,0.018079999834299088
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,31,0.019413333386182785
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,63,0.018522666146357853
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,63,0.019482667247454327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,127,0.018511999398469925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,127,0.02219199885924657
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,255,0.02418133368094762
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,255,0.022154666483402252
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,511,0.029711998999118805
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,511,0.04041066765785217
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,16383,0.21409600973129272
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,32767,0.4025919834772746
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,1023,0.061573331554730736
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,1023,0.0468800018231074
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,32767,0.7172640164693197
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,4095,0.12013333042462666
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,4095,0.19138665994008383
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,8191,0.3664106527964274
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,2047,0.07098666826883952
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,2047,0.1046346624692281
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,8191,0.21305600802103677
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,16383,0.40003732840220135
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,fp8,3,0.021183999876181286
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,16383,0.7115840117136637
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,1,0.02000533292690913
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,fp8,1,0.021205333371957142
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,7,0.01966399947802226
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,3,0.01974933346112569
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,32767,0.7885173161824545
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,fp8,7,0.02143999934196472
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,32767,1.5197973251342773
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,fp8,15,0.021183999876181286
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,15,0.01966933285196622
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,31,0.01998399943113327
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,fp8,31,0.021354667842388153
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,63,0.019776000330845516
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,127,0.02021866664290428
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,255,0.0249493345618248
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,fp8,255,0.02589866767326991
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,511,0.0420959989229838
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,fp8,511,0.031258667508761086
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,1023,0.0636053333679835
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,fp8,1023,0.0491946687301
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,fp8,63,0.021066665649414062
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,2047,0.10795199871063232
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,fp8,127,0.02386666586001714
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,fp8,2047,0.07534933090209961
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,1,0.025600001215934753
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,8191,0.3678559859593709
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,fp8,8191,0.21913067499796549
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,fp8,1,0.027802666028340656
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,4095,0.19315733512242636
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,fp8,3,0.027600000301996868
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,fp8,4095,0.12217066685358684
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,3,0.026159999271233875
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,7,0.025994665920734406
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,fp8,7,0.0277813325325648
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,15,0.026170666019121807
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,31,0.026047999660174053
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,63,0.026341333985328674
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,fp8,63,0.028005334238211315
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,127,0.027797333896160126
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,fp8,127,0.032485333581765495
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,fp8,15,0.028437333802382152
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,255,0.04444266855716705
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,fp8,255,0.039162665605545044
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,fp8,31,0.027653334041436512
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,511,0.06773866713047028
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,fp8,511,0.05487999816735586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,1023,0.10876267155011494
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,fp8,1023,0.07925333579381307
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,4095,0.365994652112325
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,fp8,4095,0.22083733479181925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,2047,0.19397334257761636
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,fp8,2047,0.12852266430854797
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,8191,0.7137493292490641
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,fp8,8191,0.4089386860529582
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,1,0.031301334500312805
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,fp8,1,0.03259200106064478
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,3,0.031119999786218006
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,fp8,3,0.032698666055997215
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,15,0.03139200061559677
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,31,0.031248000760873158
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,fp8,15,0.0325546662012736
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,fp8,31,0.03269333392381668
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,63,0.03146666785081228
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,7,0.031189332405726116
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,fp8,63,0.03274133304754893
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,fp8,7,0.0339626669883728
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,127,0.0329120010137558
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,fp8,127,0.03708266715208689
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,255,0.04964800179004669
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,fp8,255,0.04228266576925913
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,511,0.07344533503055573
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,fp8,511,0.060965334375699363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,fp8,1023,0.08497066299120586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,1023,0.1153600017229716
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,1,0.04260266820589701
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,fp8,1,0.04506133496761322
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,fp8,2047,0.13451733191808066
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,3,0.04301333427429199
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,7,0.04257066547870636
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,fp8,7,0.045050665736198425
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,fp8,3,0.04545066754023234
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,fp8,15,0.04563199977080027
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,4095,0.3729066848754883
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,fp8,4095,0.22885332504908243
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,31,0.04316799839337667
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,15,0.04306666553020477
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,fp8,31,0.046154667933781944
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,63,0.04437866806983948
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,fp8,63,0.04557333389918009
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,255,0.07495999832948048
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,fp8,255,0.07146666447321574
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,2047,0.19916266202926636
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,511,0.1185706655184428
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,fp8,511,0.09526399771372478
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,1023,0.19933332999547324
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,127,0.05373333394527435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,fp8,1023,0.1413653294245402
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,fp8,127,0.05658133327960968
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,1,0.01232533281048139
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,fp8,2047,0.2377013365427653
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,7,0.01145600030819575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,4095,0.7168213526407877
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,15,0.01137599969903628
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,3,0.012383999923865
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,fp8,4095,0.4218133290608724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,31,0.011338666081428528
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,31,0.012432000289360682
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,63,0.012309333930412928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,127,0.01145600030819575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,2047,0.37166933218638104
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,127,0.013376000026861826
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,15,0.012175999581813812
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,255,0.012666666259368261
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,255,0.015941333025693893
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,511,0.01685333376129468
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,1,0.011610666910807291
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,63,0.01145600030819575
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,1023,0.017317333569129307
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,3,0.01129066695769628
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,1023,0.01937599976857503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,2047,0.0215786670645078
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,2047,0.024432001014550526
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,4095,0.030234667162100475
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,4095,0.033957332372665405
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,8191,0.04826666911443075
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,8191,0.05327466626962026
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,16383,0.08418132861455281
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,16383,0.09296533465385437
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,32767,0.15310933192571005
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,32767,0.18516800800959268
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,65535,0.29171733061472577
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,65535,0.3529386520385742
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,1,0.011424000064531961
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,131071,0.5638560056686401
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,1,0.01257066677014033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,3,0.011445333560307821
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,7,0.012453333785136541
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,7,0.01156266654531161
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,131071,0.6861706574757894
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,3,0.012479999413092932
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,7,0.012634667257467905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,15,0.011391999820868174
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,511,0.015333333363135656
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,15,0.012736000120639801
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,31,0.011546666423479715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,31,0.012682666381200155
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,63,0.011616000284751257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,63,0.012565333396196365
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,127,0.011472000430027643
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,127,0.014005333185195923
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,255,0.012885333349307379
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,255,0.017290666699409485
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,511,0.01598400001724561
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,511,0.017360000560681026
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,1023,0.017984000345071156
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,1023,0.019909333437681198
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,2047,0.022448000808556873
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,2047,0.02495466669400533
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,4095,0.03137599925200144
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,4095,0.0346666673819224
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,8191,0.05065066615740458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,8191,0.05667200187842051
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,16383,0.08539199829101562
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,16383,0.10174399614334106
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,32767,0.15525866548220316
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,32767,0.1837386687596639
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,65535,0.2949066758155823
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,65535,0.3519253333409627
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,131071,0.5724639892578125
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,131071,0.6842506726582845
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,1,0.03875199953715006
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,fp8,1,0.03884266565243403
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,fp8,3,0.038906666139761605
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,fp8,7,0.03916800022125244
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,3,0.03846933444341024
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,7,0.03830400109291077
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,15,0.038389332592487335
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,fp8,15,0.038890667259693146
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,31,0.03846399982770284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,fp8,31,0.03878399978081385
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,fp8,63,0.039066667358080544
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,63,0.04002666721741358
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,fp8,127,0.049269333481788635
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,127,0.04710933566093445
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,255,0.06483200192451477
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,fp8,255,0.06339199841022491
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,511,0.10773866375287373
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,fp8,511,0.08713600039482117
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,1,0.06462933123111725
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,fp8,1,0.06323733429114024
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,1023,0.1933120091756185
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,fp8,1023,0.13342932860056558
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,2047,0.3670399983723958
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,3,0.06429333488146464
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,fp8,3,0.06271466612815857
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,fp8,2047,0.231605331103007
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,7,0.06433066725730896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,fp8,7,0.06297066807746887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,15,0.06440000236034393
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,fp8,15,0.0629066675901413
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,fp8,31,0.06283733248710632
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,63,0.07055466870466869
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,127,0.07483733197053273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,fp8,127,0.08669867118199666
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,255,0.11149866382280986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,31,0.06835199892520905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,fp8,255,0.10521599650382996
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,511,0.19775466124216715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,fp8,511,0.1529866655667623
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,fp8,63,0.06737599770228068
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,1023,0.36692265669504803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,fp8,1023,0.24289067586263022
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,1,0.06484800080458324
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,fp8,1,0.06574399769306183
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,2047,0.7615893681844076
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,3,0.06523733337720235
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,fp8,2047,0.43849066893259686
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,fp8,3,0.06558399895826976
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,15,0.06473066906134288
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,fp8,15,0.06564799944559734
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,fp8,7,0.06576533118883769
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,fp8,31,0.06554133196671803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,fp8,63,0.07051733136177063
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,7,0.06490133206049602
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,127,0.07691733539104462
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,fp8,127,0.08869866530100505
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,31,0.06790933509667714
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,63,0.0724373310804367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,255,0.11609599987665813
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,fp8,255,0.11050132910410564
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,511,0.19749333461125693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,1,0.1153546671072642
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,fp8,511,0.16005866726239523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,fp8,1,0.11307733257611592
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,3,0.11556800206502278
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,fp8,1023,0.24785067637761435
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,1023,0.3705226580301921
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,fp8,3,0.11244799693425496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,7,0.11573867003122966
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,fp8,7,0.11292266845703125
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,fp8,15,0.11377066373825073
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,31,0.12599466244379678
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,15,0.1202186644077301
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,fp8,63,0.12405866384506226
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,127,0.13447466492652893
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,fp8,31,0.11753599842389424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,63,0.12717866897583008
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,fp8,127,0.15747732917467752
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,1,0.011472000430027643
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,3,0.011589333415031433
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,1,0.012533333152532578
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,3,0.012634667257467905
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,fp8,255,0.19181867440541586
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,7,0.011413333316644033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,7,0.012650666137536367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,255,0.20700265963872275
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,15,0.011402666568756104
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,31,0.011557333171367645
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,31,0.012560000022252401
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,63,0.012613333761692047
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,127,0.011541333049535751
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,127,0.013786666095256805
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,255,0.012847999731699625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,255,0.014597332725922266
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,511,0.015034666905800501
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,511,0.01735466718673706
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,1023,0.01915733392039935
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,1023,0.02202133337656657
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,2047,0.027930667002995808
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,511,0.3791946570078532
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,15,0.012741333494583765
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,2047,0.03207999964555105
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,4095,0.04621866842110952
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,4095,0.05109333495299021
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,8191,0.08203733464082082
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,63,0.011834666132926941
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,16383,0.15044800440470377
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,16383,0.18347734212875366
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,32767,0.2894986669222514
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,32767,0.35199999809265137
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,65535,0.5656746625900269
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,fp8,511,0.28656532367070514
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,65535,0.6831626892089844
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,1,0.01227733368674914
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,131071,1.1211466789245605
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,1,0.013386666774749756
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,3,0.01201066623131434
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,7,0.01320533330241839
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,7,0.011952000359694162
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,15,0.012181332955757776
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,131071,1.3584213256835938
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,15,0.013125333935022354
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,31,0.011887999872366587
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,31,0.01331199953953425
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,63,0.013461332768201828
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,63,0.012069333344697952
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,127,0.012448000411192576
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,127,0.014592000593741735
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,255,0.013776000589132309
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,8191,0.09521599610646565
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,255,0.015317333241303762
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,511,0.01533866673707962
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,511,0.01781333362062772
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,1023,0.019509332875410717
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,3,0.01349866638580958
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,1023,0.022661333282788593
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,2047,0.02922133356332779
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,2047,0.03268799930810928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,4095,0.04937600096066793
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,4095,0.05422399938106537
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,8191,0.08262399832407634
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,8191,0.09915733337402344
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,16383,0.15404799580574036
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,16383,0.18309332927068075
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,32767,0.3500320116678874
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,32767,0.2918773293495178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,1023,0.7888000011444092
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,65535,0.5662933190663656
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,65535,0.6823840141296387
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,1,0.012426666915416718
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,1,0.013573333621025085
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,3,0.012106666962305704
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,131071,1.1458293596903484
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,3,0.012826666235923767
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,7,0.011946666985750198
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,15,0.012159999459981918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,fp8,1023,0.4622666835784912
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,131071,1.343664010365804
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,31,0.012149333953857422
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,15,0.013183999806642532
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,31,0.01302933320403099
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,63,0.012240000069141388
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,63,0.013088000317414602
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,127,0.015013333410024643
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,255,0.013562666873137156
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,255,0.015008000036080679
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,511,0.015504000087579092
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,511,0.017818666994571686
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,1023,0.01951466624935468
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,1023,0.022405333817005157
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,2047,0.028618666032950085
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,2047,0.032261334359645844
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,4095,0.04897066454092661
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,4095,0.05383466680844625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,8191,0.08559999863306682
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,8191,0.09994133313496907
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,16383,0.15337600310643515
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,16383,0.1832639972368876
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,32767,0.35514668623606366
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,32767,0.29133333762486774
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,65535,0.5704266627629598
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,65535,0.6846453348795573
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,1,0.011786667009194693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,1,0.012853333105643591
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,3,0.011637333780527115
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,3,0.012272000312805176
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,131071,1.120469331741333
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,7,0.011509332805871964
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,7,0.01228800043463707
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,15,0.011317333827416102
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,131071,1.3573066393534343
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,31,0.011418666690587997
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,63,0.011354666203260422
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,31,0.01267733300725619
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,15,0.012538666526476542
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,63,0.012304000556468964
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,127,0.01532799998919169
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,255,0.013722666849692663
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,255,0.015125333021084467
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,127,0.011557333171367645
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,511,0.013834666460752487
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,511,0.014853333433469137
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,7,0.013338666409254074
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,4095,0.016106666376193363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,1023,0.01394133393963178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,1023,0.014975999792416891
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,2047,0.015018666783968607
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,2047,0.01618133361140887
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,4095,0.017082666357358296
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,127,0.012560000022252401
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,8191,0.018277333428462345
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,65535,0.05086400111516317
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,8191,0.019727999965349834
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,16383,0.022869333624839783
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,16383,0.025221332907676697
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,32767,0.032431999842325844
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,32767,0.03531199942032496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,65535,0.056554665168126426
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,131071,0.08939733107884724
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,131071,0.09887466828028361
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,1,0.011354666203260422
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,1,0.012698666503032049
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,3,0.011274666835864386
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,3,0.012650666137536367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,7,0.011247999966144562
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,7,0.01268799975514412
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,15,0.01121066634853681
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,15,0.012565333396196365
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,31,0.01139733319481214
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,31,0.012805332740147909
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,63,0.011349332829316458
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,63,0.012863999853531519
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,127,0.011594666788975397
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,127,0.015589332828919092
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,255,0.013866666704416275
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,255,0.014917333920796713
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,511,0.014202666779359182
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,511,0.01526933287580808
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,1023,0.015173333386580149
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,1023,0.015856000284353893
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,2047,0.016666666915019352
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,2047,0.017386666188637417
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,4095,0.018426666657129925
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,4095,0.020021333048741024
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,8191,0.02199466774861018
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,8191,0.02439466615517934
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,16383,0.03054933249950409
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,16383,0.034160000582536064
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,32767,0.04880533119042715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,32767,0.05383466680844625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,65535,0.08411199847857158
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,65535,0.09345600008964539
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,131071,0.1545973320802053
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,131071,0.18617600202560425
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,1,0.013023999830087027
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,1,0.013658666362365087
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,3,0.013023999830087027
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,3,0.013674666484196981
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,7,0.012954667210578918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,7,0.013754667093356451
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,15,0.012928000340859095
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,15,0.013450667262077332
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,63,0.013002666334311167
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,31,0.013104000439246496
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,31,0.013530666629473368
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,63,0.013493333011865616
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,127,0.014192000031471252
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,127,0.015103999525308609
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,255,0.016122666498025257
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,255,0.01463466634353002
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,511,0.016528000434239704
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,511,0.018885333091020584
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,1023,0.021317332983016968
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,1023,0.023445333043734234
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,2047,0.03711999952793121
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,2047,0.03530666728814443
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,4095,0.06118399898211161
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,4095,0.057909334699312844
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,8191,0.10891200105349223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,8191,0.10152000188827515
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,16383,0.19614400466283163
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,16383,0.1853546698888143
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,1,0.010698666175206503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,32767,0.4043999910354614
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,1,0.012159999459981918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,3,0.011264000087976456
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,3,0.01219733307758967
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,7,0.01128000020980835
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,7,0.012245333443085352
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,32767,0.35370667775472003
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,15,0.011823999385039011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,31,0.011194666226704916
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,15,0.012650666137536367
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,65535,0.6977386474609375
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,131071,1.5759520530700684
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,31,0.013167999684810638
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,63,0.012618667135636011
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,63,0.011173332730929056
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,255,0.013631999492645264
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,255,0.015333333363135656
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,65535,0.745568037033081
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,511,0.015802666544914246
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,511,0.01482133318980535
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,1023,0.015856000284353893
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,127,0.014282666146755219
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,1023,0.017050666113694508
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,127,0.011237333218256632
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,131071,1.3717652956644695
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,8191,0.029701332251230877
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,2047,0.019573333362738293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,4095,0.021664001047611237
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,2047,0.017194667210181553
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,4095,0.02403733382622401
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,32767,0.09490666786829631
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,8191,0.03429333368937174
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,16383,0.053770666321118675
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,16383,0.04886400202910105
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,32767,0.08331733445326488
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,65535,0.15377599994341531
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,65535,0.18380266427993774
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,131071,0.29128533601760864
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,131071,0.3539786736170451
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,1,0.014080000420411428
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,1,0.015322666615247726
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,3,0.01423466702302297
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,3,0.01545599972208341
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,15,0.014389333625634512
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,7,0.014549333602190018
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,7,0.015493333339691162
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,15,0.01551466683546702
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,31,0.014277332772811254
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,31,0.015381333728631338
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,63,0.01423466702302297
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,63,0.015184000134468079
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,127,0.016970666746298473
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,127,0.014789332946141561
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,255,0.017407999684413273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,255,0.017360000560681026
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,511,0.018170667191346485
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,511,0.020197333147128422
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,1023,0.03692266593376795
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,1023,0.0272533322374026
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,2047,0.0587360014518102
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,2047,0.04414933423201243
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,8191,0.19033600886662802
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,8191,0.11757333079973857
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,16383,0.2132693330446879
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,4095,0.06922133266925812
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,4095,0.10345600048700969
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,16383,0.36694931983947754
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,1,0.019754666835069656
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,32767,0.7154773076375326
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,3,0.02160000056028366
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,32767,0.4043360153834025
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,7,0.019626667102177937
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,1,0.021104000508785248
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,3,0.019658666104078293
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,65535,1.528704007466634
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,65535,0.7835840384165446
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,7,0.021344001094500225
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,15,0.021114667256673176
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,31,0.0198186660806338
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,15,0.019551999866962433
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,31,0.021055998901526134
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,127,0.020549333343903225
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,127,0.0239680012067159
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,255,0.024725332856178284
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,255,0.025439999997615814
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,511,0.04222933451334635
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,63,0.019962667177120846
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,511,0.034688000877698265
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,63,0.021183999876181286
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,1023,0.04967466493447622
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,1023,0.06393066545327504
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,2047,0.07509866853555043
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,2047,0.10682666301727295
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,8191,0.3673653205235799
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,8191,0.21964265902837118
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,4095,0.12168533603350322
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,16383,0.7197279930114746
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,4095,0.19375999768575033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,1,0.030970667799313862
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,3,0.031311998764673867
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,fp8,3,0.03254399945338567
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,fp8,1,0.03236266722281774
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,16383,0.40614934762318927
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,fp8,7,0.032357332607110344
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,fp8,15,0.032474666833877563
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,7,0.030933332939942677
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,31,0.031311998764673867
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,15,0.031157332162062328
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,fp8,31,0.03254933406909307
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,63,0.03128000100453695
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,127,0.03183999905983607
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,fp8,127,0.03679466744263967
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,fp8,63,0.03268799930810928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,255,0.05029866596062978
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,fp8,255,0.042133331298828125
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,511,0.07239466905593872
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,fp8,511,0.06073066592216492
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,1023,0.1141866644223531
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,fp8,1023,0.08444799979527791
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,2047,0.19923200209935507
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,1,0.01129066695769628
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,fp8,8191,0.4158453146616618
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,fp8,2047,0.13385599851608276
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,3,0.012330666184425354
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,7,0.011359999577204386
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,1,0.012074666718641916
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,8191,0.7257386843363444
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,15,0.01138666644692421
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,15,0.012309333930412928
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,31,0.011312000453472137
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,63,0.01121066634853681
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,63,0.01227733368674914
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,127,0.011183999478816986
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,127,0.014021333307027817
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,255,0.015856000284353893
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,255,0.012736000120639801
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,31,0.013007999708255133
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,511,0.01525866612792015
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,511,0.016741332908471424
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,1023,0.01754666616519292
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,1023,0.019632000476121902
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,2047,0.021727999051411945
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,3,0.011296000331640244
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,2047,0.023887999355793
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,fp8,4095,0.2286293307940165
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,7,0.012304000556468964
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,4095,0.030378667016824085
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,4095,0.0341386670867602
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,4095,0.3717493216196696
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,8191,0.04794666667779287
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,8191,0.053269331653912864
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,16383,0.08468266328175862
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,32767,0.15238400300343832
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,32767,0.18595200777053833
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,65535,0.2918720046679179
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,131071,0.5677546660105387
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,131071,0.6840906937917074
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,1,0.03839999934037527
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,16383,0.09512000282605489
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,3,0.03817066550254822
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,fp8,1,0.03877866764863332
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,fp8,3,0.03892799963553747
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,65535,0.35421331723531085
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,fp8,7,0.03862400104602178
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,7,0.038373333712418876
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,fp8,15,0.03972266614437103
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,15,0.038133333126703896
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,31,0.03835200021664301
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,63,0.04009066770474116
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,fp8,63,0.039359999199708305
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,fp8,31,0.038949333131313324
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,127,0.045647998650868736
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,fp8,127,0.04888000090916952
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,255,0.06486399968465169
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,fp8,255,0.06369066735108693
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,511,0.10654933253924052
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,fp8,511,0.08827199538548787
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,1023,0.1922773321469625
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,fp8,1023,0.13141333063443503
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,2047,0.3650346597035726
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,fp8,2047,0.23007466395696005
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,1,0.06519466638565063
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,fp8,4095,0.41722134749094647
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,fp8,3,0.06588266789913177
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,fp8,1,0.0653546651204427
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,3,0.06493866443634033
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,4095,0.7200906276702881
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,7,0.06464533507823944
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,15,0.06503466765085857
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,fp8,7,0.0651039977868398
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,31,0.06693866848945618
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,63,0.07167466481526692
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,fp8,31,0.06519466638565063
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,127,0.07606400052706401
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,fp8,15,0.06554666658242543
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,fp8,127,0.08898666501045227
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,255,0.11332799990971883
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,fp8,255,0.1106719970703125
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,511,0.19818667570749918
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,fp8,511,0.15812266866366068
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,1,0.012223999947309494
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,3,0.011610666910807291
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,fp8,63,0.06905599931875865
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,7,0.011461333682139715
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,7,0.012789333860079447
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,15,0.012842666357755661
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,15,0.011418666690587997
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,31,0.011488000551859537
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,31,0.01240533341964086
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,63,0.01146666705608368
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,63,0.012416000167528788
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,1,0.012586666891972223
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,127,0.011600000162919363
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,127,0.013429333766301474
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,255,0.012885333349307379
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,1023,0.36814932028452557
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,255,0.014858666807413101
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,511,0.01479999969402949
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,3,0.012666666259368261
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,511,0.017242666333913803
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,1023,0.019333332777023315
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,2047,0.027808000644048054
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,2047,0.03188266605138779
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,4095,0.046426668763160706
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,4095,0.051258668303489685
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,8191,0.08165866633256276
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,8191,0.09247466921806335
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,fp8,1023,0.24743467569351196
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,16383,0.15077867110570273
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,32767,0.28917332490285236
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,16383,0.1832159956296285
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,32767,0.34880534807840985
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,65535,0.6857546965281168
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,131071,1.1194667021433513
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,131071,1.3541173934936523
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,1023,0.021888000269730885
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,65535,0.5661813418070475
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,fp8,2047,0.4426826635996501
VLLM,0.12.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,2047,0.7607200145721436
