framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,window_size,beam_width,attn_dtype,kv_cache_dtype,step,latency
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,1,0.024165332317352295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,1,0.023823998868465424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,3,0.024266667664051056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,3,0.023786666492621105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,7,0.024714666108290356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,15,0.02569599946339925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,7,0.02500266581773758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,15,0.026191999514897663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,31,0.031583999594052635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,31,0.03224000086386999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,63,0.03190399954716364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,63,0.03249600032965342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,127,0.03417066733042399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,127,0.03278933217128118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,255,0.04770133395989736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,255,0.04021333406368891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,511,0.07643199960390727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,511,0.06553600231806438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,1023,0.1848906675974528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,1023,0.1090773344039917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,2047,0.22460800409317017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,2047,0.35549867153167725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,4095,0.69486403465271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,4095,0.42882664998372394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,1,0.007871999715765318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,1,0.00914666677514712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,3,0.007829333345095316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,3,0.00901333304742972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,7,0.009178666397929192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,15,0.009039999917149544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,31,0.00810666692753633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,31,0.009162666896979014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,63,0.008703999842206636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,63,0.009813333551088968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,127,0.008912000184257826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,7,0.007807999849319458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,127,0.011349332829316458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,255,0.00898133342464765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,255,0.011333333949247995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,511,0.009904000287254652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,15,0.007829333345095316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,511,0.011503999431928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,1023,0.016197333733240765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,1023,0.01580799991885821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,2047,0.02759466568628947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,2047,0.027488000690937042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,4095,0.03620799879233042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,16383,0.11963733037312825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,4095,0.03436800092458725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,8191,0.06484800080458324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,1,0.008005333443482717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,16383,0.08379200100898743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,1,0.009109333157539368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,3,0.009354666496316591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,3,0.008016000191370646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,7,0.008058666562040647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,7,0.009056000038981438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,15,0.008101333553592363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,15,0.009194666519761086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,31,0.008245333408315977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,31,0.00927466650803884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,8191,0.04781333108743032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,63,0.009205333267649015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,63,0.010010666524370512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,127,0.009136000027259191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,127,0.011482667177915573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,255,0.009279999881982803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,511,0.010351999973257383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,255,0.011760000139474869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,511,0.011610666910807291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,1023,0.026869334280490875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,1023,0.025834667185942333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,2047,0.031354665756225586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,2047,0.03054933249950409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,4095,0.05346666773160299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,16383,0.19285867611567178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,16383,0.12443733215332031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,1,0.04274133344491323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,4095,0.04061333338419596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,1,0.04397333165009817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,8191,0.06605866551399231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,8191,0.1071626643339793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,3,0.04303466777006785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,3,0.04393066465854645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,7,0.044581333796183266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,7,0.045381332437197365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,15,0.04593066871166229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,15,0.047269334395726524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,31,0.05783466498057047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,31,0.05914133290449778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,63,0.06039999922116598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,63,0.05950400233268738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,127,0.06612800061702728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,127,0.06170133252938589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,255,0.08468799789746602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,255,0.08016000191370647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,511,0.13928533593813577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,511,0.12284266948699951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,1023,0.3569440046946208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,1023,0.20678933461507162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,1,0.00873066671192646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,2047,0.6975999673207601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,2047,0.43148799737294513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,1,0.008805333326260248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,3,0.008821333448092142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,3,0.008896000062425932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,7,0.008842666943868002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,7,0.008976000050703684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,15,0.00922133338948091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,15,0.008725333337982496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,31,0.009722666814923286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,31,0.009637333452701569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,63,0.011418666690587997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,63,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,127,0.011349332829316458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,127,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,255,0.011381333072980246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,255,0.011407999942700068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,511,0.013536000003417334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,511,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,1023,0.018805333723624546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,1023,0.01793066660563151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,2047,0.05249600112438202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,2047,0.02882133424282074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,4095,0.0953386624654134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,4095,0.06181866427262624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,8191,0.1827146609624227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,8191,0.11223466197649638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,16383,0.2160586714744568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,16383,0.35841067632039386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,1,0.08020799855391185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,1,0.08177599807580312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,3,0.0800799975792567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,3,0.08255999783674876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,7,0.084197332461675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,7,0.08614400029182434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,15,0.0879200001557668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,31,0.11354133486747742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,15,0.0888853371143341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,31,0.11109866698582967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,63,0.12008000413576762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,63,0.11784000198046367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,511,0.26336532831192017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,127,0.12442133824030559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,127,0.12332800030708313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,255,0.1583199997742971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,255,0.15102400382359824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,511,0.23312532901763916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,1023,0.39722132682800293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,float16,1,0.15532799561818442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,fp8,1,0.1590026617050171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,1023,0.6970346768697103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,float16,3,0.1556106706460317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,fp8,3,0.15914666652679443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,float16,7,0.1623199979464213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,fp8,7,0.16566399733225504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,float16,15,0.1827626625696818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,fp8,15,0.17364267508188883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,float16,31,0.22513065735499063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,fp8,31,0.22482667366663614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,float16,63,0.23388799031575522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,float16,127,0.23812800645828247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,fp8,63,0.2352693279584249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,fp8,127,0.23655466238657633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,float16,255,0.3039253354072571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,fp8,255,0.295413335164388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,float16,1,0.30580800771713257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,fp8,1,0.30988266070683795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,float16,3,0.31996800502141315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,fp8,3,0.3116053342819214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,float16,7,0.3484213352203369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,fp8,7,0.33552531401316327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,float16,15,0.36506132284800213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,fp8,15,0.3707786798477173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,float16,31,0.4426399866739909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,fp8,31,0.4459306796391805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,float16,63,0.45929598808288574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,fp8,63,0.4603360096613566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,1,0.014090667168299357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,1,0.014261333892742792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,float16,127,0.4703146616617839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,fp8,127,0.46557867527008057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,3,0.014058666924635569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,3,0.01404800017674764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,7,0.014021333307027817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,7,0.014314666390419006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,15,0.014426667243242264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,15,0.014597332725922266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,31,0.01545599972208341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,31,0.015381333728631338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,63,0.01870399961868922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,63,0.018719999740521114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,127,0.01894933357834816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,127,0.01884799947341283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,255,0.01926933353145917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,255,0.019274666905403137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,511,0.023685333629449207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,511,0.02216533323129018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,1023,0.04370133578777313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,1023,0.03342399994532267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,2047,0.09848533074061076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,2047,0.05937600135803223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,4095,0.1835040052731832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,4095,0.11655466755231221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,8191,0.3540693521499634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,8191,0.21569067239761353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,16383,0.42134400208791095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,16383,0.7044266859690348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,float16,1,0.6649706761042277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,fp8,1,0.6727893352508545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,float16,3,0.6719626585642496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,fp8,3,0.6830666859944662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,float16,7,0.6853866577148438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,fp8,7,0.701573371887207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,float16,15,0.7229013442993164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,fp8,15,0.7300586700439453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,float16,31,0.878058671951294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,fp8,31,0.8916479746500651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,float16,63,0.9106293519337972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,fp8,63,0.9121812979380289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,float16,1,1.3322933514912922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,fp8,1,1.3560746510823567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,float16,3,1.3281386693318684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,1,0.01392000044385592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,fp8,3,1.3604159355163574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,1,0.014218666901191076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,float16,7,1.3726080258687336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,3,0.013957332819700241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,fp8,7,1.3958667119344075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,float16,15,1.4287573496500652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,3,0.0141546664138635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,fp8,15,1.4623200098673503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,7,0.014426667243242264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,float16,31,1.74834140141805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,fp8,31,1.76311461130778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,7,0.01470400020480156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,15,0.01488000030318896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,15,0.014938666174809137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,31,0.018090666582187016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,31,0.018320000420014065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,63,0.018378666291634243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,63,0.018624000251293182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,127,0.0184906671444575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,127,0.01858666663368543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,255,0.022837333381175995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,255,0.021759999295075733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,511,0.04284800092379252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,511,0.0337119996547699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,1023,0.09803199768066406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,1023,0.0587360014518102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,2047,0.18310399850209555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,2047,0.11769599715868632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,4095,0.353493332862854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,4095,0.21877866983413696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,8191,0.43829333782196045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,8191,0.7218720118204752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,1,0.023599999646345775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,1,0.024256000916163128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,3,0.02430933217207591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,3,0.02510400116443634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,7,0.025253333151340485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,7,0.025759999950726826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,15,0.031557333966096245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,31,0.03193599979082743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,31,0.03190399954716364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,63,0.03408533334732056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,63,0.03215999901294708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,15,0.032058666149775185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,127,0.04765866696834564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,127,0.040175999204317726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,255,0.07609599828720093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,255,0.06630399823188782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,511,0.18577067057291666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,511,0.1095306674639384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,1023,0.2244853377342224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,1023,0.35602664947509766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,2047,0.6941280364990234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,2047,0.4291626612345378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,1,0.008954666554927826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,1,0.009002666920423508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,4095,0.842192014058431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,4095,1.3792959849039714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,3,0.009056000038981438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,3,0.009279999881982803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,7,0.008858666444818178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,7,0.009056000038981438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,15,0.00914666677514712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,15,0.009242666885256767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,31,0.009919999788204828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,255,0.01139733319481214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,31,0.009797333429257074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,63,0.011541333049535751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,63,0.01146666705608368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,127,0.011312000453472137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,127,0.011349332829316458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,255,0.01173866664369901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,511,0.016341333587964375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,511,0.015791999797026317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,1023,0.027509334186712902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,1023,0.027258666853109997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,2047,0.03617066641648611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,2047,0.034448000291983284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,4095,0.06329066554705302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,4095,0.04867733518282572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,8191,0.11352533102035522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,8191,0.08517866333325703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,16383,0.20430932442347208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,3,0.009370666618148485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,1,0.009072000160813332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,7,0.009173333023985228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,16383,0.15428800384203592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,1,0.009359999870260557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,3,0.009189333145817121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,7,0.008992000172535578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,15,0.009434666484594345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,15,0.009370666618148485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,31,0.009813333551088968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,31,0.010159999753038088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,63,0.01157333329319954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,63,0.011823999385039011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,127,0.011535999675591787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,127,0.01180800050497055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,255,0.012144000579913458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,255,0.011706666400035223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,511,0.026687999566396076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,511,0.026047999660174053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,1023,0.031199999153614044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,1023,0.030437332888444264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,2047,0.05275199810663859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,2047,0.04207466542720795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,4095,0.1076746682325999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,4095,0.06869333485762279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,8191,0.1922880013783773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,8191,0.13125333189964294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,16383,0.36088534196217853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,16383,0.23803200324376425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,1,0.04242666562398275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,1,0.04386133452256521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,3,0.04455466568470001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,3,0.04563733438650767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,7,0.04599999884764353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,7,0.04753600060939789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,15,0.05784533421198527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,15,0.0582826683918635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,31,0.06117333471775055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,31,0.058389330903689064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,63,0.0668693333864212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,63,0.06266133487224579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,127,0.08558932940165202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,127,0.08026133477687836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,255,0.1399679978688558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,255,0.12312533458073933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,511,0.35763732592264813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,511,0.20789867639541626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,1023,0.43483201662699383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,1023,0.6980213324228922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,1,0.008602666358153025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,1,0.008799999952316284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,2047,0.841866652170817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,3,0.008570666735370954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,2047,1.3772266705830891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,3,0.00897066667675972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,7,0.00854399986565113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,7,0.008997333546479544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,15,0.009408000235756239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,15,0.009573333586255709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,31,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,31,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,63,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,63,0.011312000453472137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,127,0.011434666812419891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,127,0.011535999675591787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,255,0.013151999562978745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,255,0.013269333789745966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,511,0.018709332992633183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,511,0.018112000077962875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,1023,0.051829333106676735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,1023,0.028570666909217834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,2047,0.09496000409126282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,2047,0.061978667974472046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,4095,0.11249066392580669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,4095,0.18204800287882486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,8191,0.35493866602579754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,8191,0.21361599365870157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,16383,0.7078293164571127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,16383,0.42079468568166095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,1,0.08012799918651581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,1,0.0830506682395935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,3,0.08374399940172832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,3,0.08572799960772197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,7,0.0881173312664032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,7,0.08881066242853801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,15,0.11672000090281169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,15,0.11097066601117452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,31,0.12134933471679688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,31,0.11897599697113037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,63,0.12500799695650736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,63,0.122597336769104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,127,0.1592693328857422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,127,0.15218666195869446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,255,0.2655946612358093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,255,0.24103466669718424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,511,0.6986933549245199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,511,0.401589314142863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,1023,1.3827412923177083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,1023,0.8412480354309082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,1,0.1556533376375834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,fp8,1,0.1597813367843628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,3,0.16815465688705444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,fp8,3,0.16790932416915894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,7,0.18636800845464072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,fp8,7,0.1827626625696818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,15,0.2285919984181722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,fp8,15,0.23152534166971842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,31,0.23548799753189087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,fp8,31,0.23341333866119385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,63,0.23954667647679648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,fp8,63,0.23933867613474527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,127,0.3063093423843384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,fp8,127,0.29782400528589886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,255,0.5072746674219767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,fp8,255,0.463210662206014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,1,0.3396586577097575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,fp8,1,0.34251201152801514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,7,0.3664960066477458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,3,0.34726401170094806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,fp8,3,0.35869868596394855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,fp8,7,0.376693328221639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,15,0.44735999902089435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,fp8,15,0.45746131738026935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,31,0.46031999588012695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,fp8,31,0.46302398045857746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,63,0.4728693167368571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,fp8,63,0.4678399960199992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,1,0.013679999858140945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,1,0.014096000542243322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,127,0.5993546644846598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,fp8,127,0.5872533321380615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,3,0.01379199946920077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,3,0.0143306665122509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,7,0.014064000298579534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,7,0.014335999886194864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,15,0.015002666662136713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,15,0.015471999843915304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,31,0.01838933303952217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,31,0.018687999496857326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,63,0.018543999642133713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,127,0.019039999693632126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,63,0.018751999984184902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,127,0.01884799947341283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,255,0.023413332800070446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,255,0.022389332453409832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,511,0.043151999513308205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,511,0.033957332372665405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,1023,0.09799466530481975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,1023,0.05990933378537496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,2047,0.1835199991861979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,2047,0.11528533697128296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,4095,0.3535733222961426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,4095,0.2166986664136251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,8191,0.41836798191070557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,8191,0.6982506910959879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,16383,0.8298239707946777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,16383,1.4019893010457356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,float16,1,0.6730720202128092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,fp8,1,0.6875572999318441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,float16,3,0.6868693033854166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,fp8,3,0.7105066776275635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,float16,7,0.726032018661499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,fp8,7,0.7464906374613444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,float16,15,0.8964213530222574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,fp8,15,0.9085333347320557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,float16,31,0.913424015045166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,fp8,31,0.9130826791127523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,float16,63,0.9305120309193929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,fp8,63,0.9333706696828207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,float16,1,1.3397119839986165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,fp8,1,1.376479943593343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,float16,3,1.3757707277933757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,fp8,3,1.4136266708374023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,float16,7,1.4352533022562664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,fp8,7,1.4781866073608398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,float16,15,1.7697067260742188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,fp8,15,1.8101439476013184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,float16,31,1.8322347005208333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,1,0.01368533323208491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,1,0.013872000078360239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,3,0.01402666668097178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,fp8,31,1.8311680157979329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,3,0.01423466702302297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,7,0.014805333067973455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,7,0.01462399959564209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,15,0.017871999492247898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,31,0.017845333864291508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,15,0.018138666947682697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,63,0.018394666413466137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,31,0.018085333208243053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,63,0.017973333597183228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,127,0.02232533444960912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,127,0.02163200080394745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,255,0.04244266450405121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,255,0.03297599901755651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,511,0.09803199768066406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,511,0.05902933577696482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,1023,0.11732799808184306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,1023,0.18288000424702963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,2047,0.22129066785176596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,2047,0.353002667427063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,4095,0.42787198225657147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,4095,0.695466677347819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,1,0.018826667219400406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,8191,0.8586453596750895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,8191,1.4960373242696126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,1,0.019050666441520054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,3,0.01894933357834816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,3,0.01931200052301089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,7,0.019482667247454327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,7,0.019968000551064808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,15,0.02029866725206375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,15,0.02042666698495547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,31,0.025258667767047882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,31,0.025199999411900837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,63,0.02508266766866048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,63,0.025631998976071674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,127,0.025722667574882507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,127,0.025594666600227356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,255,0.037402667105197906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,255,0.030447999636332195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,511,0.06004266440868378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,511,0.05083199838797251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,1023,0.14165866374969482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,1023,0.08457600076993306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,2047,0.269487996896108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,2047,0.17040000359217325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,1,0.008047999814152718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,1,0.009066666786869368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,4095,0.3227573235829671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,3,0.008005333443482717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,4095,0.5252853234608968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,3,0.009296000003814697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,15,0.009098666409651438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,7,0.00786666696270307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,7,0.009093333035707474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,15,0.008223999912540117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,31,0.008133333176374435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,31,0.009248000259200731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,63,0.010170666500926018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,63,0.008938666433095932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,127,0.008826666822036108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,127,0.011440000186363855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,255,0.009029333169261614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,255,0.0116799995303154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,511,0.01007466639081637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,511,0.011493333925803503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,1023,0.01624533285697301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,1023,0.015925332903862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,2047,0.02722666660944621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,2047,0.027072000006834667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,4095,0.03480533262093862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,4095,0.034373333056767784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,8191,0.05916266640027364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,8191,0.047370667258898415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,16383,0.1071626643339793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,16383,0.08290666838486989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,1,0.007797333101431529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,1,0.009173333023985228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,3,0.007946666950980822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,3,0.009312000125646591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,7,0.007871999715765318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,7,0.009189333145817121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,15,0.008016000191370646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,15,0.009317333499590555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,31,0.007978666573762894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,31,0.009322666873534521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,63,0.008890666688481966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,63,0.010079999764760336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,127,0.009103999783595404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,127,0.011424000064531961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,255,0.009296000003814697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,255,0.011690666278203329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,511,0.010154666379094124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,511,0.011744000017642975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,1023,0.02608533451954524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,1023,0.02588266630967458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,2047,0.03036266565322876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,2047,0.03010133405526479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,4095,0.04709866642951965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,4095,0.04041599979003271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,8191,0.0869706670443217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,8191,0.063701331615448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,16383,0.15847999850908914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,16383,0.11481600006421407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,1,0.03357866654793421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,1,0.03422400106986364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,3,0.03395200024048487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,3,0.03440533330043157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,7,0.03502399971087774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,7,0.03557866563399633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,15,0.035946667194366455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,15,0.037087999284267426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,31,0.04463466505209605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,31,0.04569066564242045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,63,0.04593066871166229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,63,0.04626133541266123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,127,0.05202666421731313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,127,0.046469335754712425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,255,0.06646933158238728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,255,0.061861331264177956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,511,0.10839999715487163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,511,0.0949173370997111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,1023,0.26968000332514447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,1023,0.1590826710065206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,2047,0.527728001276652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,1,0.008703999842206636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,1,0.008661333471536636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,2047,0.32867199182510376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,3,0.008645333349704742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,3,0.008837333569924036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,7,0.00898133342464765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,7,0.008976000050703684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,15,0.00873066671192646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,15,0.008842666943868002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,31,0.009519999846816063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,31,0.009525333220760027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,63,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,63,0.011402666568756104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,127,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,127,0.011472000430027643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,255,0.011477333803971609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,255,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,511,0.013162666310866674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,511,0.012810666114091873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,1023,0.018250666558742523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,1023,0.01823466643691063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,2047,0.0373279998699824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,2047,0.029018667836983997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,4095,0.07641066610813141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,4095,0.05589866638183594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,8191,0.1418560047944387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,8191,0.10734400153160095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,16383,0.27750933170318604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,16383,0.19871999820073447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,1,0.06229866544405619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,1,0.06369600196679433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,3,0.061861331264177956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,3,0.0636053333679835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,7,0.06411733229955037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,7,0.06584000090758006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,15,0.06663466493288676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,15,0.06816000243028005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,31,0.08514666557312012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,31,0.08527466654777527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,63,0.09207466244697571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,63,0.0864586631457011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,127,0.09459732969601949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,127,0.09365333120028178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,255,0.12146666646003723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,255,0.11615999539693196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,511,0.20240533351898193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,511,0.17869333426157633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,1023,0.5243839820226034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,1023,0.30157333612442017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,float16,1,0.11723732948303223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,fp8,1,0.12010666728019714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,float16,3,0.11828266580899556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,fp8,3,0.12129066387812297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,float16,7,0.12382400035858154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,fp8,7,0.1269760032494863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,float16,15,0.1328053375085195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,fp8,15,0.13085333506266275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,float16,31,0.16941867272059122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,fp8,31,0.1667893330256144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,float16,63,0.17722133795420328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,fp8,63,0.17830399672190347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,fp8,1,0.23437867561976114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,float16,127,0.18161600828170776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,fp8,127,0.1811573306719462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,float16,255,0.2302453319231669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,fp8,255,0.2237280011177063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,float16,1,0.22818666696548462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,float16,3,0.2322559952735901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,fp8,3,0.23644800980885824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,float16,7,0.2539199988047282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,fp8,7,0.24959999322891235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,float16,15,0.27636800209681195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,fp8,15,0.27024000883102417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,float16,127,0.35237332185109455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,float16,31,0.3345973491668701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,fp8,31,0.3365653355916341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,1,0.013776000589132309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,float16,63,0.3460586468378703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,fp8,63,0.3489813407262166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,fp8,127,0.3541333278020223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,1,0.013754667093356451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,3,0.013653332988421122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,3,0.013936000565687815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,7,0.013760000467300415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,7,0.014175999909639359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,15,0.013738666971524557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,15,0.014170666535695394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,31,0.013898666948080063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,31,0.014730667074521383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,63,0.015354666858911514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,63,0.015728000551462173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,127,0.01569066693385442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,127,0.015935999651749928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,255,0.0161920003592968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,255,0.015882667154073715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,511,0.018858666221300762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,511,0.018170667191346485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,1023,0.03408533334732056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,1023,0.025983999172846477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,2047,0.07869866490364075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,2047,0.048538664976755776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,4095,0.15083733201026917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,4095,0.09299733241399129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,8191,0.2871786753336589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,8191,0.18397333224614462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,16383,0.5655999978383383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,16383,0.3556319872538249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,float16,1,0.4884959856669108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,fp8,1,0.47751466433207196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,float16,3,0.5051360130310059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,fp8,3,0.4943679968516032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,float16,7,0.5167200167973837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,fp8,7,0.5290079911549886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,float16,15,0.5415626764297485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,fp8,15,0.5539626677831014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,float16,31,0.6615466674168905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,fp8,31,0.6715253194173177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,float16,63,0.6847573121388754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,fp8,63,0.6863626639048258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,float16,1,0.994645357131958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,fp8,1,1.0207839806874592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,float16,3,0.9979466597239176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,fp8,3,1.0236639976501465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,float16,7,1.0329973697662354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,fp8,7,1.0510079860687256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,float16,15,1.0838560263315837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,fp8,15,1.0935946305592854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,3,0.013082666943470636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,float16,31,1.3067519664764404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,1,0.013269333789745966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,1,0.01332266628742218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,fp8,31,1.3383253415425618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,3,0.01313599944114685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,7,0.013866666704416275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,7,0.013354666531085968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,15,0.013194666554530462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,15,0.013295999417702356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,31,0.014752000570297241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,31,0.015263999501864115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,63,0.01515199989080429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,63,0.015253332753976187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,127,0.01544533297419548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,127,0.01525866612792015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,255,0.01811733345190684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,255,0.017685333887736004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,511,0.0339626669883728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,511,0.025685332715511322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,1023,0.07801066835721333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,1023,0.04806933303674062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,2047,0.15060800313949585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,2047,0.09568533301353455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,4095,0.287173330783844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,4095,0.18515199422836304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,8191,0.5700693527857462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,1,0.018506667266289394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,1,0.019205333044131596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,3,0.019248000035683315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,3,0.019914666811625164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,7,0.02004266654451688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,8191,0.38504000504811603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,7,0.020527999848127365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,15,0.02470933397610982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,15,0.025066666305065155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,31,0.025072000920772552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,31,0.025120000044504803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,63,0.025429333249727886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,63,0.0252960001428922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,127,0.038202665746212006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,127,0.030405332644780476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,255,0.05935466786225637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,255,0.05108266572157542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,511,0.1415839989980062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,511,0.08452266454696655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,1023,0.2698880036671956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,1023,0.17130666971206665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,2047,0.3259626626968384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,2047,0.5251359939575195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,7,0.008805333326260248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,4095,1.0385119915008545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,1,0.009018666421373686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,4095,0.635807991027832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,1,0.009061333412925402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,3,0.00892800030608972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,3,0.00901333304742972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,7,0.009290666629870733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,15,0.00898133342464765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,15,0.00921066664159298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,31,0.009824000298976898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,31,0.009765333185593287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,63,0.011445333560307821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,63,0.011450666934251785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,127,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,127,0.011621333658695221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,255,0.011488000551859537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,255,0.011584000041087469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,511,0.015840000162522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,511,0.015802666544914246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,1023,0.027210667729377747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,1023,0.02699733277161916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,2047,0.03507733345031738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,2047,0.0339626669883728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,4095,0.059792002042134605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,4095,0.04727466901143392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,8191,0.10719466209411621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,8191,0.08349333206812541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,16383,0.1918826699256897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,16383,0.14972266554832458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,1,0.009162666896979014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,1,0.009226666763424873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,3,0.008789333204428354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,3,0.009317333499590555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,7,0.008837333569924036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,7,0.009279999881982803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,15,0.009173333023985228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,15,0.009216000015536943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,31,0.010064000263810158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,31,0.01009599988659223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,63,0.01139733319481214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,63,0.011637333780527115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,127,0.011498666057984034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,127,0.011600000162919363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,255,0.012191999703645706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,255,0.011589333415031433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,511,0.026170666019121807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,511,0.02554133286078771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,1023,0.030037333567937214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,1023,0.03012799968322118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,2047,0.04645866652329763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,2047,0.04054400076468786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,4095,0.08707732955614726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,4095,0.06615466872851054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,8191,0.15718932946523032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,8191,0.11814933021863301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,16383,0.29149333635965985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,16383,0.22082134087880453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,1,0.033359999457995095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,1,0.03454933315515518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,3,0.034586665530999504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,3,0.03573333223660787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,7,0.0359946663180987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,7,0.03666666646798452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,15,0.04496000210444132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,15,0.04558933277924856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,31,0.04572799801826477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,31,0.04560000201066335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,63,0.051856001218159996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,63,0.04615999758243561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,127,0.06714666883150737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,127,0.06218666831652323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,255,0.10868266224861145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,255,0.09505599737167358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,511,0.27210666735967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,511,0.16005333264668783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,2047,1.0384000142415364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,1,0.008752000207702318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,1023,0.5275466839472452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,1023,0.3304426670074463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,1,0.008517333616813024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,3,0.008661333471536636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,3,0.008869333192706108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,7,0.008639999975760778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,2047,0.6369066635767618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,7,0.009002666920423508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,15,0.009509333098928133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,15,0.009461333354314169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,31,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,31,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,63,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,63,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,127,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,127,0.01145600030819575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,255,0.012981332838535309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,255,0.013072000195582708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,511,0.017968000223239262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,511,0.0182239996890227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,1023,0.03364799916744232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,1023,0.028463999430338543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,2047,0.07517866790294647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,2047,0.05406400064627329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,4095,0.14219199617703757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,4095,0.10365333159764607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,8191,0.27476799488067627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,8191,0.19662932554880777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,1,0.06157866617043813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,16383,0.5475093523661295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,16383,0.3856426477432251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,3,0.06388799846172333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,1,0.06423999865849812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,3,0.06618133187294006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,7,0.06614399949709575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,7,0.06919999917348225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,15,0.08569600184758504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,15,0.08532800277074178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,31,0.09334400296211243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,31,0.08747733632723491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,63,0.09595200419425964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,63,0.0937493344148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,127,0.11629333098729451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,127,0.12283733487129211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,255,0.20509866873423258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,255,0.17972799142201742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,511,0.5281973282496134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,511,0.30421332518259686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,1,0.11737599968910217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,1023,0.635477344195048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,fp8,1,0.12139200170834859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,1023,1.0418293476104736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,3,0.1237440009911855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,fp8,3,0.12723199526468912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,7,0.13773866494496664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,fp8,7,0.133242666721344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,15,0.17258665959040323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,fp8,15,0.17231466372807822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,fp8,31,0.17751999696095785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,31,0.17801600694656372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,63,0.18316799402236938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,127,0.23375467459360758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,fp8,63,0.1814133326212565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,fp8,127,0.22617600361506143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,fp8,255,0.35014931360880536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,255,0.38577067852020264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,1,0.24680533011754355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,fp8,1,0.24261866013209024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,3,0.2646399935086568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,fp8,3,0.2645600040753682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,7,0.27767467498779297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,fp8,7,0.285045325756073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,15,0.34069331487019855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,fp8,15,0.34327467282613117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,1,0.013631999492645264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,31,0.34783466657002765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,fp8,31,0.3477120002110799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,63,0.357258677482605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,fp8,63,0.35523732503255206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,127,0.4533066749572754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,1,0.013749333719412485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,3,0.013647999614477158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,3,0.013151999562978745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,fp8,127,0.4435306787490845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,7,0.013354666531085968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,7,0.013354666531085968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,15,0.013728000223636627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,15,0.01381333296497663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,31,0.015210667004187902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,31,0.01568000018596649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,63,0.015626666446526844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,63,0.015669333438078564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,127,0.01603200038274129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,127,0.015967999895413715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,255,0.01884799947341283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,255,0.01870399961868922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,511,0.03429333368937174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,4095,0.28753066062927246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,511,0.026704000929991405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,1023,0.07929599781831105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,1023,0.04957866668701172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,2047,0.15085867047309875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,2047,0.0936853289604187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,4095,0.1866933306058248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,8191,0.35920532544453937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,8191,0.5608319838841757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,16383,1.120026667912801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,16383,0.7079946994781494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,float16,1,0.5084480047225952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,fp8,1,0.5189813375473022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,float16,3,0.5224639972050985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,fp8,3,0.5376693407694498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,float16,7,0.5487039883931478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,fp8,7,0.5599199930826823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,float16,15,0.6746293703715006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,fp8,15,0.6802026430765787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,float16,31,0.6935093402862549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,fp8,31,0.6886773109436035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,float16,63,0.7054773171742758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,fp8,63,0.7030879656473795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,float16,1,1.0101439952850342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,fp8,1,1.0289973417917888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,float16,3,1.0380160013834636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,fp8,3,1.0580213069915771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,float16,7,1.0907519658406575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,fp8,7,1.1140373547871907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,1,0.013007999708255133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,1,0.013023999830087027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,float16,15,1.3420799573262532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,3,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,fp8,15,1.3630879720052083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,3,0.01322666679819425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,7,0.013173333058754602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,7,0.013125333935022354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,15,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,15,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,31,0.014981333166360855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,31,0.015237333873907724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,float16,31,1.380176067352295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,fp8,31,1.3700106938680012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,63,0.015087999403476715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,63,0.01523200049996376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,127,0.018170667191346485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,127,0.017701332767804463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,255,0.03399466723203659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,255,0.0260959987839063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,511,0.077824001510938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,511,0.04855466882387797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,1023,0.1499733328819275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,1023,0.10027733445167542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,2047,0.2874079942703247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,2047,0.19047466913859049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,4095,0.3639039993286133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,4095,0.5605493386586508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,8191,0.7425173123677572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,8191,1.127237319946289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,1,0.017616000026464462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,1,0.017738666385412216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,3,0.017488000293572743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,3,0.018021332720915478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,7,0.018144000321626663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,7,0.01854933301607768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,15,0.018394666413466137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,15,0.01850133389234543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,31,0.021525333325068157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,31,0.021850667893886566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,63,0.02170666555563609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,63,0.02213866760333379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,127,0.022170667846997578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,127,0.022346665461858112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,255,0.03249066571394602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,255,0.025920001169045765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,511,0.053344001372655235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,511,0.04374399781227112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,1023,0.12219732999801636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,1023,0.0735040009021759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,2047,0.2363040049870809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,2047,0.14615999658902487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,7,0.007786666974425316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,1,0.008042666440208754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,4095,0.28570665915807086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,4095,0.4591093460718791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,1,0.00902399979531765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,3,0.007807999849319458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,3,0.009248000259200731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,7,0.008853333070874214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,127,0.008933333059151968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,15,0.008133333176374435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,15,0.009296000003814697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,31,0.008154666672150293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,31,0.009296000003814697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,511,0.011509332805871964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,63,0.008714666590094566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,63,0.00985599992175897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,127,0.011306667079528173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,255,0.008976000050703684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,255,0.011685332904259363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,511,0.010149333626031876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,1023,0.01598400001724561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,1023,0.015775999675194424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,2047,0.01738133281469345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,2047,0.016906666258970898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,4095,0.030053332448005676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,4095,0.030016000072161358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,8191,0.044010668992996216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,8191,0.040218666195869446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,1,0.007823999971151352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,16383,0.0860533316930135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,1,0.009136000027259191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,16383,0.061621333161989846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,3,0.007823999971151352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,3,0.009279999881982803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,7,0.008117333054542542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,7,0.009328000247478485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,15,0.008000000069538752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,127,0.011653333902359009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,15,0.009093333035707474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,31,0.008143999924262365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,31,0.00919999989370505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,63,0.0086666668454806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,63,0.0099093330403169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,127,0.009130666653315226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,255,0.009237333511312803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,255,0.01163200040658315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,511,0.010015999898314476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,511,0.011626667032639185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,1023,0.025802666942278545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,1023,0.025519999365011852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,2047,0.030229332546393078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,2047,0.030000001192092896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,4095,0.04339733223120371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,4095,0.039936001102129616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,8191,0.08624000350634257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,8191,0.06214400132497152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,16383,0.14991999665896097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,1,0.02886933336655299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,16383,0.11260799566904704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,1,0.029520000020662945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,3,0.028981332977612812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,3,0.029343999922275543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,7,0.029765332738558452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,7,0.03030399978160858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,15,0.030938667555650074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,15,0.03124266614516576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,31,0.03821333249409994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,31,0.03902400036652883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,63,0.03845866769552231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,63,0.039173332353432976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,127,0.04375466704368591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,127,0.039594667653242745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,255,0.05710933109124502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,255,0.05275199810663859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,1023,0.13327466448148093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,511,0.09216533104578654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,511,0.08027199904123943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,1023,0.22825066248575845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,1,0.00874133345981439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,1,0.014831999937693277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,2047,0.2763200004895528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,2047,0.44093334674835205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,3,0.008698666468262672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,3,0.014794666320085526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,7,0.00879466657837232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,7,0.015082667271296183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,15,0.008746666833758354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,15,0.014938666174809137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,31,0.009541333342591921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,31,0.015450666348139444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,63,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,63,0.01661866654952367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,127,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,127,0.01979200045267741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,255,0.011525332927703857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,255,0.019834666202465694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,2047,0.0322826678554217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,511,0.01332266628742218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,511,0.01979200045267741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,1023,0.018085333208243053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,1023,0.023029332359631855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,2047,0.02867199977238973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,4095,0.06789333124955495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,4095,0.054917335510253906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,8191,0.12575466434160867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,8191,0.1077280044555664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,1,0.05276266733805338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,3,0.05397866666316986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,16383,0.24594666560490927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,16383,0.19883199532826742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,1,0.053472002347310386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,3,0.05287466446558634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,7,0.05422399938106537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,7,0.05596266686916351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,15,0.05658133327960968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,15,0.05762133498986562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,31,0.07067733506361644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,31,0.07115200161933899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,63,0.07748800019423167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,63,0.07250133156776428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,127,0.08102933565775554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,127,0.0796319991350174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,255,0.10336533188819885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,255,0.09788266817728679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,511,0.1701493263244629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,float16,1,0.09892266988754272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,511,0.14933866262435913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,1023,0.44095468521118164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,1023,0.2534346580505371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,fp8,1,0.10211199522018433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,float16,3,0.09985066453615825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,fp8,3,0.10221333305040996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,float16,7,0.10393599669138591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,fp8,7,0.10602133472760518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,float16,15,0.1086079974969228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,fp8,15,0.10913599530855815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,float16,31,0.14337066809336343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,fp8,31,0.13948800166447958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,float16,63,0.1491200029850006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,fp8,63,0.1497599979241689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,float16,127,0.1532533367474874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,float16,255,0.19350399573644003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,fp8,127,0.15135467052459717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,fp8,255,0.18677866458892822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,float16,1,0.1929439902305603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,fp8,1,0.19797333081563315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,fp8,7,0.2067413330078125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,float16,3,0.19376534223556519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,fp8,3,0.1981226603190104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,float16,7,0.20828266938527426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,float16,15,0.23000532388687134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,fp8,15,0.218560000260671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,float16,31,0.28039999802907306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,fp8,31,0.2840213378270467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,1,0.013327999661366144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,float16,63,0.2879679997762044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,1,0.01358933374285698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,fp8,63,0.29258666435877484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,float16,127,0.2975733280181885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,3,0.013776000589132309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,fp8,127,0.2967466711997986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,3,0.013290667285521826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,7,0.013199999928474426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,7,0.013701333353916803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,15,0.01331199953953425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,15,0.013386666774749756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,31,0.01381333296497663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,31,0.013525333255529404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,63,0.01543466622630755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,63,0.015610666324694952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,127,0.015669333438078564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,127,0.01589866727590561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,255,0.016202667107184727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,1023,0.025663999219735462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,255,0.015935999651749928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,511,0.018816000471512478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,511,0.01826133330663045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,1023,0.029535998900731403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,2047,0.07643733421961467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,2047,0.046725332736968994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,4095,0.09296000003814697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,4095,0.1409173309803009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,8191,0.27270400524139404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,8191,0.1722453236579895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,16383,0.35022934277852374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,16383,0.5336480140686035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,float16,1,0.391269326210022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,fp8,1,0.3910986582438151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,float16,3,0.41495466232299805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,fp8,3,0.3997013171513875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,float16,7,0.43230934937795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,fp8,7,0.43908798694610596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,float16,15,0.4548693497975667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,fp8,15,0.46319464842478436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,float16,31,0.5532053311665853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,fp8,31,0.5574026505152384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,float16,63,0.5718026558558146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,fp8,63,0.5728373527526855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,float16,1,0.836949348449707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,fp8,1,0.8469173113505045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,float16,3,0.8387306531270345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,fp8,3,0.8543679714202881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,float16,7,0.8568960030873617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,fp8,7,0.8786880175272623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,1,0.013221333424250284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,1,0.012773333738247553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,float16,15,0.8992640177408854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,3,0.013072000195582708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,3,0.01249066616098086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,7,0.013013333082199097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,7,0.013354666531085968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,15,0.012778667112191519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,15,0.012896000097195307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,31,0.013839999834696451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,31,0.01370666672786077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,63,0.013914667069911957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,fp8,15,0.9199946721394857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,127,0.014277332772811254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,63,0.014032000054915747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,127,0.014159999787807465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,float16,31,1.0920960108439128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,255,0.016437333077192307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,255,0.01621866722901662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,511,0.029482667644818623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,fp8,31,1.1159839630126953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,511,0.023056000471115112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,1023,0.07358933488527934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,4095,0.17056000232696533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,1023,0.044821331898371376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,2047,0.09091732899347942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,2047,0.1393066644668579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,4095,0.27475200096766156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,1,0.01736533393462499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,1,0.01758933315674464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,3,0.01802666609485944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,8191,0.36032533645629883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,8191,0.5143093268076578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,3,0.01807466646035512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,7,0.01810666670401891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,7,0.01842133328318596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,15,0.02145066608985265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,127,0.025701334079106648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,15,0.02165866643190384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,31,0.02163733293612798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,31,0.02178666740655899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,511,0.12236799796422322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,63,0.02197333425283432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,63,0.02180800090233485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,127,0.03215999901294708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,255,0.05312533179918925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,255,0.04407466451327006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,511,0.07426666716734569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,2047,0.2909119923909505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,1023,0.1483573317527771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,1,0.009088000282645226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,1023,0.2363626758257548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,2047,0.45985599358876544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,1,0.00871999996403853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,7,0.008832000195980072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,3,0.00873066671192646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,3,0.009066666786869368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,4095,0.5674026807149252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,4095,0.9063733418782552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,7,0.009173333023985228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,15,0.008821333448092142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,15,0.009322666873534521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,31,0.009578666960199675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,31,0.009808000177145004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,63,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,63,0.011509332805871964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,127,0.011530666301647821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,127,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,1023,0.016837333639462788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,255,0.011557333171367645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,255,0.011333333949247995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,511,0.015781333049138386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,511,0.015498666713635126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,1023,0.01701333373785019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,2047,0.029626667499542236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,2047,0.030053332448005676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,4095,0.04340800146261851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,4095,0.039813332259655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,8191,0.08660800258318584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,8191,0.06411733229955037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,1,0.009808000177145004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,1,0.009173333023985228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,3,0.008922666932145754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,16383,0.14871467153231302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,16383,0.11804800232251485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,3,0.00960533320903778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,7,0.009045333291093508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,7,0.009183999771873156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,15,0.009002666920423508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,15,0.009285333255926767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,31,0.009717333440979322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,31,0.009824000298976898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,63,0.01192533348997434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,63,0.011306667079528173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,127,0.011429333438475927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,127,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,255,0.011551999797423681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,255,0.011381333072980246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,511,0.025642665723959606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,511,0.025616000096003216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,1023,0.02980799973011017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,1023,0.029834667841593426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,2047,0.04354666670163473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,2047,0.039733332892258964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,4095,0.08610133330027263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,4095,0.06270933151245117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,8191,0.15040533741315207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,8191,0.11825600266456604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,1,0.0284853329261144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,1,0.029264000554879505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,3,0.029450667401154835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,16383,0.2770666678746541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,16383,0.2141759991645813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,7,0.03068266560633977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,3,0.030634666482607525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,7,0.03134933362404505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,15,0.03826133410135905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,15,0.03853866706291834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,31,0.03846933444341024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,31,0.03877866764863332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,63,0.044400001565615334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,63,0.03933866570393244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,127,0.05700799822807312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,127,0.05286933481693268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,255,0.0922933320204417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,255,0.08017600079377492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,511,0.2282080054283142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,511,0.13551466663678488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,1023,0.4417653481165568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,1023,0.2784213423728943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,1,0.014474666366974512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,1,0.014757333944241205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,3,0.014432000617186228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,2047,0.535536011060079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,3,0.015103999525308609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,2047,0.8689493338267008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,7,0.014309333016475042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,7,0.014917333920796713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,15,0.014901333798964819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,15,0.015247999380032221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,31,0.016095999628305435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,31,0.01651200031240781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,63,0.01956266661485036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,127,0.01959466685851415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,63,0.019813333948453266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,127,0.01972266659140587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,255,0.019802667200565338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,255,0.01979200045267741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,511,0.0227360005180041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,511,0.022986667851607006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,1023,0.03716266651948293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,1023,0.032405334214369454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,2047,0.08083199958006541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,2047,0.05578133463859558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,4095,0.14358400305112204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,4095,0.10854933659235637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,8191,0.2720639904340108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,8191,0.20122667153676352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,1,0.05380799869696299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,1,0.0524479995171229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,3,0.05564799904823303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,16383,0.5274080038070679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,3,0.053717335065205894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,16383,0.3877813418706258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,7,0.056202664971351624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,7,0.05758399764696757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,15,0.07193066676457723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,15,0.07130133112271626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,31,0.07857066889603932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,31,0.07213866710662842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,63,0.08147199948628743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,63,0.07985066870848338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,127,0.10382399956385295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,127,0.09891733527183533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,255,0.17132800817489624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,255,0.15093333522478738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,511,0.25545599063237506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,511,0.4422293504079183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,1,0.09902399778366089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,fp8,1,0.1014400025208791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,3,0.10341333349545796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,1023,0.532965342203776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,fp8,3,0.10700800021489461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,1023,0.8708586692810059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,7,0.1104746659596761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,fp8,7,0.111653337876002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,15,0.14524799585342407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,fp8,15,0.14145599802335104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,31,0.14949867129325867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,63,0.15411733587582907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,fp8,31,0.14985600113868713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,fp8,63,0.1516746679941813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,127,0.19689599672953287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,fp8,127,0.18869866927464804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,255,0.32816533247629803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,fp8,255,0.2911413311958313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,1,0.19953600565592447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,fp8,7,0.2360639969507853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,fp8,1,0.20013866821924844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,3,0.21768534183502197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,fp8,3,0.21454399824142456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,7,0.23145065704981485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,15,0.2858026623725891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,fp8,15,0.2882080078125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,fp8,31,0.29357866446177167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,31,0.2938613295555115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,63,0.2996266682942708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,1,0.012752000242471695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,1,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,fp8,63,0.29579732815424603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,3,0.012784000486135483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,127,0.3812853495279948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,fp8,127,0.3683520158131917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,3,0.013290667285521826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,7,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,63,0.015615999698638916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,7,0.013093333691358566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,15,0.013349333157142004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,15,0.01358933374285698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,31,0.015466666469971338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,31,0.015599999576807022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,63,0.015749332805474598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,127,0.016122666498025257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,127,0.015754666179418564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,255,0.018618666877349217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,255,0.018474667022625606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,511,0.031002665559450786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,511,0.026133333643277485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,1023,0.07808533310890198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,1023,0.04754666487375895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,2047,0.14205867052078247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,2047,0.09297600388526917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,4095,0.2728426655133565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,4095,0.1730133295059204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,8191,0.3497120141983032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,8191,0.5344693263371786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,16383,0.6870666344960531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,16383,1.059775988260905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,float16,1,0.4262933333714803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,fp8,1,0.43457599480946857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,float16,3,0.4376000165939331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,fp8,3,0.4460959831873576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,float16,7,0.45977600415547687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,fp8,7,0.4692586660385132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,float16,15,0.5646239916483561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,float16,31,0.5801759958267212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,fp8,15,0.5738186836242676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,fp8,31,0.5759040117263794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,float16,63,0.5899626811345419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,fp8,63,0.5830453236897787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,float16,3,0.8670079708099365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,fp8,3,0.8911146322886149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,float16,7,0.9126506646474203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,fp8,7,0.9312160015106201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,fp8,1,0.8624320030212402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,float16,1,0.8453599611918131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,1,0.012202666451533636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,float16,15,1.1223093668619792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,1,0.012538666526476542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,3,0.012645332763592402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,3,0.012527999778588613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,7,0.012533333152532578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,7,0.012719999998807907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,15,0.013781332721312841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,fp8,15,1.1407893498738606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,fp8,31,1.1527040004730225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,15,0.013973332941532135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,31,0.01404800017674764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,63,0.014325333138306936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,127,0.01648533344268799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,float16,31,1.1530133088429768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,31,0.013957332819700241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,63,0.014192000031471252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,127,0.0164533331990242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,255,0.02962133288383484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,255,0.023200000325838726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,511,0.07186666627724965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,511,0.04353600243727366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,1023,0.14019200205802917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,1023,0.09283199906349182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,2047,0.2746400038401286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,2047,0.1748746633529663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,1,0.013781332721312841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,4095,0.354042649269104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,1,0.014368000129858652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,4095,0.5374826590220133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,3,0.014015999933083853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,3,0.014021333307027817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,8191,0.711855967839559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,7,0.014378666877746582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,7,0.014783999572197596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,31,0.018277333428462345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,15,0.01504533365368843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,127,0.018453333526849747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,15,0.015018666783968607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,8191,1.0216106573740642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,31,0.01791999985774358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,63,0.01836266616980235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,63,0.01912533367673556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,127,0.018677332748969395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,255,0.022698665658632915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,255,0.022181332111358643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,2047,0.11735467116038005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,511,0.03408533334732056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,511,0.04252799848715464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,1023,0.09794132908185323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,1023,0.05905066430568695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,2047,0.1836479902267456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,4095,0.3542293310165405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,4095,0.22041600942611694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,1,0.00784533346692721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,1,0.009072000160813332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,3,0.007850666840871176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,3,0.009290666629870733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,8191,0.4405279954274495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,31,0.007994666695594788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,7,0.00785600021481514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,8191,0.720853328704834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,7,0.009237333511312803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,15,0.007813333223263422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,255,0.00921066664159298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,15,0.009125333279371262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,31,0.009237333511312803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,63,0.00996800015370051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,511,0.011418666690587997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,63,0.00879466657837232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,127,0.00879466657837232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,127,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,2047,0.01699200024207433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,255,0.01128000020980835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,511,0.0102186668664217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,1023,0.015728000551462173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,1023,0.015685333559910457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,2047,0.01706133286158244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,4095,0.027477333943049114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,4095,0.027349332968393963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,8191,0.038405333956082664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,8191,0.03670933345953623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,16383,0.07548266649246216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,16383,0.05586666862169901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,1,0.007733333234985669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,1,0.008992000172535578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,3,0.008016000191370646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,3,0.009029333169261614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,31,0.008186666915814081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,32767,0.12437333663304646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,32767,0.09506133198738098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,7,0.007962666451931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,7,0.008965333302815756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,15,0.007696000238259633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,15,0.00891733355820179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,31,0.00919999989370505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,63,0.008789333204428354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,63,0.009962666779756546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,127,0.008912000184257826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,127,0.011525332927703857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,255,0.00927466650803884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,255,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,511,0.009893333539366722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,511,0.011578666667143503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,1023,0.016602666427691776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,1023,0.01590399940808614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,2047,0.027808000644048054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,2047,0.027765333652496338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,4095,0.03625600039958954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,4095,0.03435199956099192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,8191,0.06332266827424367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,8191,0.04826133449872335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,16383,0.11340799927711487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,16383,0.08260266482830048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,1,0.02363733450571696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,1,0.024400000770886738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,3,0.02386666586001714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,32767,0.14784533778826395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,3,0.024277334411938984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,7,0.024800000091393787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,32767,0.20669333140055338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,7,0.02500266581773758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,15,0.025701334079106648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,15,0.026208000878492992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,31,0.031530665854612984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,31,0.032042667269706726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,255,0.047824000318845115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,63,0.03198933353026708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,63,0.032842665910720825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,127,0.03408000121514002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,1023,0.18508267402648926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,127,0.03275199979543686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,255,0.04045866678158442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,511,0.07603733241558075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,511,0.06594133377075195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,1023,0.11081600189208984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,2047,0.2253119945526123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,2047,0.35575465361277264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,1,0.00795199970404307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,1,0.009306666751702627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,3,0.008063999935984612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,3,0.009269333134094873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,4095,0.43010131518046063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,4095,0.6967573165893555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,7,0.007887999837597212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,7,0.009178666397929192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,15,0.008058666562040647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,15,0.009077333534757296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,31,0.008266666904091835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,31,0.009279999881982803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,63,0.009962666779756546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,63,0.008879999940594038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,127,0.009125333279371262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,127,0.011690666278203329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,255,0.009178666397929192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,255,0.011776000261306763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,511,0.01028266673286756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,2047,0.030458666384220123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,511,0.011706666400035223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,1023,0.027141332626342773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,1023,0.025946666797002155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,2047,0.032085334261258446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,4095,0.05318933228651682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,4095,0.04057066639264425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,8191,0.10803733269373576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,1,0.04331733286380768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,8191,0.06594133377075195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,16383,0.19371734062830606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,16383,0.12272533774375916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,1,0.04370133578777313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,7,0.04471466441949209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,3,0.04330666859944662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,3,0.04419733087221781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,32767,0.36380799611409503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,32767,0.22519999742507935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,7,0.04569066564242045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,15,0.0466186652580897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,63,0.0591839998960495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,15,0.046997333566347756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,31,0.05806933343410492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,31,0.05919999877611796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,63,0.06150400141874949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,127,0.06650666892528534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,127,0.06186666587988535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,255,0.0849226713180542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,255,0.07987200220425923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,511,0.13915733496348062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,511,0.12320533394813538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,1023,0.3571733236312866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,1023,0.20803199211756387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,1,0.08117866516113281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,1,0.08237866560618083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,2047,0.4327840010325114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,2047,0.6991679668426514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,3,0.08063466846942902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,3,0.0830506682395935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,7,0.08450133601824443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,7,0.08656533559163411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,15,0.08810133735338847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,15,0.08846933643023173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,31,0.1135040024916331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,31,0.1116480032602946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,63,0.12065600355466206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,63,0.1188213328520457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,127,0.12462400396664937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,127,0.12365866700808208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,255,0.15715733170509338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,255,0.15119466185569763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,511,0.2627840042114258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,511,0.2357599933942159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,float16,1,0.15590399503707886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,fp8,1,0.16031466921170553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,float16,3,0.1562026639779409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,fp8,3,0.15891200304031372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,float16,7,0.16300800442695618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,fp8,7,0.1662826637427012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,float16,15,0.1837973395983378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,fp8,15,0.17394665877024332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,float16,31,0.22632000843683878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,fp8,31,0.22779732942581177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,float16,63,0.23162132501602173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,fp8,63,0.23337066173553467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,1,0.008976000050703684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,float16,127,0.23824532826741537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,fp8,127,0.23963199059168497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,3,0.008687999720374743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,1,0.008602666358153025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,float16,255,0.3039039969444275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,3,0.00884799969693025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,fp8,255,0.2964800000190735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,7,0.008816000074148178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,7,0.008869333192706108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,15,0.008837333569924036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,15,0.00867733359336853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,31,0.009658666948477427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,31,0.009648000200589498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,63,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,63,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,127,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,127,0.01126933346192042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,255,0.011413333316644033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,255,0.011530666301647821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,2047,0.052069331208864846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,511,0.013424000392357508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,511,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,1023,0.018592000007629395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,1023,0.01838933303952217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,2047,0.028698667883872986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,4095,0.06150400141874949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,4095,0.09637332955996196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,8191,0.18167465925216675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,8191,0.11238933602968852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,16383,0.2184000015258789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,16383,0.3593813180923462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,32767,0.6072213252385458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,float16,1,0.30477333068847656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,fp8,1,0.3140160044034322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,float16,3,0.3225546677907308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,32767,0.7884053389231364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,float16,7,0.34929601351420086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,fp8,3,0.3153653343518575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,fp8,7,0.3377973238627116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,float16,15,0.36394667625427246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,fp8,15,0.36885865529378253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,float16,31,0.4449919859568278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,fp8,31,0.4509066740671794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,fp8,63,0.4628053506215413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,float16,63,0.4587999979654948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,float16,127,0.4699573516845703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,fp8,127,0.46902398268381756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,float16,1,0.6661119858423868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,fp8,1,0.670197327931722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,float16,3,0.6733813285827637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,fp8,3,0.684613307317098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,float16,7,0.6920373439788818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,fp8,7,0.7046293417612711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,float16,15,0.7257546583811442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,fp8,15,0.7327253023783366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,1,0.014138666292031607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,1,0.014250667144854864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,3,0.014325333138306936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,3,0.014218666901191076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,7,0.014256000518798828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,7,0.014378666877746582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,15,0.0145066666106383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,15,0.014474666366974512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,31,0.01533866673707962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,float16,31,0.8753493626912435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,31,0.015599999576807022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,63,0.018976000448067982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,63,0.018805333723624546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,fp8,63,0.9184106985727946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,fp8,31,0.888709306716919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,127,0.018837332725524902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,float16,63,0.9097066720326742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,127,0.01887999971707662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,255,0.019280000279347103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,255,0.019237333287795384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,511,0.02346133440732956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,2047,0.09860266248385112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,511,0.022367998957633972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,1023,0.04358399907747904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,1023,0.03414933383464813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,2047,0.0591893345117569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,4095,0.18412800629933676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,4095,0.11636267105738322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,8191,0.3556480010350545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,8191,0.216922660668691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,1,0.013781332721312841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,1,0.014064000298579534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,3,0.014175999909639359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,16383,0.7020906607309977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,15,0.01803733284274737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,3,0.014416000495354334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,31,0.018181333939234417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,31,0.01794133335351944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,63,0.01817600056529045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,7,0.014602666099866232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,127,0.022821334501107533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,16383,0.4245813290278117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,255,0.042725334564844765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,255,0.03358400116364161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,7,0.015040000279744467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,15,0.018229333062966663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,63,0.018298666924238205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,127,0.021514666577180225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,511,0.09802132844924927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,511,0.059077332417170204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,1023,0.18336532513300577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,1023,0.11789333820343018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,2047,0.22206934293111166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,2047,0.3528053363164266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,1,0.00884799969693025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,4095,0.4293866554896037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,1,0.009050666665037474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,4095,0.6959199905395508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,3,0.008874666566650072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,3,0.009018666421373686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,8191,0.8592639764149984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,7,0.009045333291093508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,7,0.009056000038981438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,15,0.008986666798591614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,8191,1.4284799893697102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,15,0.00916800027092298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,31,0.010357333347201347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,31,0.009648000200589498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,63,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,63,0.011413333316644033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,127,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,127,0.011333333949247995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,255,0.011541333049535751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,1023,0.016458666572968166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,255,0.011413333316644033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,511,0.015450666348139444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,511,0.01563199982047081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,1023,0.016757333030303318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,2047,0.027248000105222065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,2047,0.027514666318893433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,4095,0.03803733239571253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,4095,0.03682133307059606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,8191,0.07544533411661784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,8191,0.054645334680875145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,16383,0.12601066629091898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,16383,0.09895466764767964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,32767,0.22696532805760702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,32767,0.17470399538675943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,1,0.009039999917149544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,1,0.008778666456540426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,3,0.009072000160813332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,3,0.008896000062425932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,7,0.008954666554927826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,7,0.009077333534757296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,15,0.008949333180983862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,15,0.009125333279371262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,31,0.009679999823371569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,31,0.009839999799927076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,63,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,63,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,127,0.011424000064531961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,127,0.011461333682139715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,255,0.011493333925803503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,255,0.011343999455372492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,511,0.016197333733240765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,511,0.015696000307798386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,1023,0.027477333943049114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,1023,0.027263998985290527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,2047,0.03558400024970373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,2047,0.03435199956099192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,4095,0.0658133327960968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,4095,0.04834666848182678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,8191,0.11307733257611592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,8191,0.08456533153851827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,16383,0.20226667324701944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,16383,0.15413332978884378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,1,0.023520000278949738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,1,0.02422400067249934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,3,0.024458666642506916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,32767,0.2860426704088847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,32767,0.38395198186238605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,3,0.025125332176685333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,7,0.025194667279720306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,7,0.02586666742960612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,15,0.03162133445342382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,63,0.032032000521818794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,15,0.03166933357715607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,31,0.03202133377393087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,31,0.03204799940188726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,63,0.033626665671666466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,127,0.04788800080617269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,127,0.041082667807737984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,255,0.07588799794514973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,255,0.06598400076230367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,511,0.18595200777053833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,511,0.11065066854159038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,1023,0.3559253215789795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,1023,0.22567999362945557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,2047,0.6958080132802328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,2047,0.43082666397094727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,1,0.009072000160813332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,1,0.00919999989370505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,4095,0.8421013355255127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,3,0.008986666798591614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,4095,1.378053347269694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,3,0.009173333023985228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,7,0.009098666409651438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,7,0.009248000259200731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,63,0.011605333536863327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,15,0.00916800027092298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,127,0.011754666765530905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,31,0.00997866690158844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,15,0.009583999713261923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,63,0.011488000551859537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,31,0.00996800015370051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,127,0.011610666910807291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,255,0.011957333733638128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,255,0.011855999628702799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,511,0.026736001173655193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,511,0.02611733227968216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,1023,0.031130666534105938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,4095,0.06900799771149953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,1023,0.030095999439557392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,2047,0.05201066533724467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,2047,0.041759997606277466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,4095,0.10705600182215373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,16383,0.3598506848017375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,1,0.04266133407751719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,8191,0.19158933560053507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,8191,0.13079466422398886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,16383,0.2371573249499003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,1,0.04420800010363261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,3,0.0444213350613912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,3,0.045610666275024414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,7,0.046298667788505554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,7,0.04773866633574168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,32767,0.45057066281636554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,32767,0.7020906607309977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,15,0.05861333509286245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,15,0.05876266459623972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,31,0.06196266909440359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,31,0.058559998869895935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,127,0.08050666749477386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,63,0.067221333583196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,63,0.06358399987220764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,127,0.08549867073694865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,255,0.1404800017674764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,255,0.12349866827329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,511,0.20826667547225952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,511,0.35816534360249835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,1,0.0809333324432373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,1023,0.43638400236765545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,1023,0.6998293399810791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,3,0.08692800005276997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,1,0.08352532982826233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,3,0.08327466746171315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,7,0.08800533413887024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,2047,0.841375986735026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,7,0.09019200007120769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,15,0.11288000146547954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,15,0.11669333775838216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,2047,1.3786452611287434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,31,0.12328533331553142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,31,0.119077334801356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,63,0.12537599603335062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,63,0.12295466661453247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,127,0.16049066185951233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,127,0.15396266182263693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,255,0.2656159996986389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,255,0.24091200033823648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,fp8,1,0.15958399573961893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,511,0.7007573445638021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,511,0.4020853439966838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,1,0.15612266461054483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,3,0.1693333387374878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,fp8,3,0.16916267077128092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,7,0.18665599822998047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,fp8,7,0.18297600746154785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,15,0.22884267568588257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,fp8,15,0.2329919934272766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,63,0.24183466037114462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,31,0.23753599325815836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,fp8,31,0.23692800601323447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,1,0.008821333448092142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,fp8,63,0.23826134204864502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,1,0.008725333337982496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,127,0.30719467004140216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,fp8,127,0.2967360019683838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,3,0.008709333216150602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,7,0.009066666786869368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,3,0.00878399983048439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,255,0.5119573275248209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,7,0.008821333448092142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,fp8,255,0.4638773202896118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,15,0.009290666629870733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,15,0.009594666461149851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,31,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,31,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,63,0.011391999820868174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,63,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,127,0.01157333329319954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,127,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,255,0.013306666165590286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,255,0.013167999684810638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,511,0.018522666146357853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,511,0.018357332795858383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,1023,0.05190933247407278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,1023,0.028517333169778187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,2047,0.09490133325258891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,2047,0.061568001906077065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,4095,0.18145600954691568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,4095,0.11213866869608562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,8191,0.35342931747436523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,8191,0.2178666591644287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,16383,0.42398401101430255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,16383,0.7095093727111816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,1,0.34065600236256915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,fp8,1,0.34832000732421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,32767,1.213653326034546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,32767,1.551029364267985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,fp8,3,0.3587520122528076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,3,0.3521973292032878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,7,0.36977601051330566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,fp8,7,0.3771680196126302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,15,0.45420801639556885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,fp8,15,0.4616853396097819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,fp8,31,0.4628533522288005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,fp8,63,0.4717706839243571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,31,0.46505598227183026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,63,0.47465066115061444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,fp8,127,0.5851893424987793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,127,0.6016960144042969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,float16,1,0.6796267032623291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,fp8,1,0.6967893441518148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,float16,3,0.6914506753285726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,fp8,3,0.7152746518452963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,float16,15,0.8952693144480387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,fp8,15,0.9165813128153483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,float16,7,0.7279307047526041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,fp8,7,0.753216028213501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,1,0.013744000345468521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,1,0.014170666535695394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,3,0.014010666559139887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,float16,31,0.9255946477254232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,7,0.013999999811251959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,3,0.014522666732470194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,7,0.01451733335852623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,fp8,31,0.9249227046966553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,15,0.015290666371583939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,15,0.015696000307798386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,31,0.018618666877349217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,float16,63,0.9326026439666748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,fp8,63,0.9353066285451254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,31,0.018746666610240936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,63,0.018885333091020584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,63,0.018837332725524902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,127,0.01903466631968816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,127,0.019088000059127808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,255,0.02363733450571696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,255,0.02218666672706604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,511,0.04369066655635834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,511,0.033946665624777474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,1023,0.09853866696357727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,1023,0.05964266757170359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,2047,0.18353599309921265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,2047,0.11555733283360799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,4095,0.3537546793619792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,4095,0.21617066860198975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,1,0.013104000439246496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,8191,0.4188426733016968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,8191,0.6969813505808512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,1,0.013306666165590286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,3,0.013104000439246496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,3,0.013023999830087027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,7,0.013264000415802002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,7,0.01339200014869372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,16383,0.8310560385386149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,63,0.015077333897352219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,16383,1.395290692647298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,15,0.01350933313369751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,15,0.01333333303531011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,31,0.014949332922697067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,31,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,255,0.01775466650724411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,127,0.01534933348496755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,63,0.015317333241303762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,255,0.018021332720915478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,127,0.01543466622630755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,511,0.034117333590984344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,511,0.025797332326571148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,1023,0.07804266611735027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,1023,0.04808000226815542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,2047,0.15093866984049478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,2047,0.09644266963005066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,1,0.007797333101431529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,4095,0.18836265802383423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,4095,0.28809599081675213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,1,0.009018666421373686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,3,0.007696000238259633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,8191,0.3841866652170817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,3,0.008858666444818178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,8191,0.5712800025939941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,7,0.007781333600481351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,7,0.009173333023985228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,15,0.007738666608929634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,15,0.008949333180983862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,31,0.00810666692753633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,31,0.009056000038981438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,63,0.008746666833758354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,63,0.009861333295702934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,127,0.008826666822036108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,127,0.011381333072980246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,255,0.00878399983048439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,255,0.011413333316644033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,511,0.009733333562811216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,511,0.011519999553759893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,1023,0.015717333803574245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,1023,0.015546667079130808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,2047,0.016832000265518825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,16383,0.04719999929269155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,2047,0.01647466669480006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,4095,0.01732800031701724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,4095,0.016837333639462788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,8191,0.0348693331082662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,8191,0.03405333310365677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,16383,0.061903998255729675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,1,0.007887999837597212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,32767,0.10729599992434184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,32767,0.08243200182914734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,1,0.009253333633144697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,3,0.007802666475375493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,3,0.009306666751702627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,7,0.007727999861041705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,7,0.00919999989370505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,15,0.007818666597207388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,15,0.009136000027259191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,31,0.008112000301480293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,31,0.009290666629870733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,63,0.0086666668454806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,63,0.009946666657924652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,127,0.008869333192706108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,127,0.011823999385039011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,255,0.009008000294367472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,255,0.011509332805871964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,511,0.00985599992175897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,4095,0.03477866699298223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,511,0.011605333536863327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,1023,0.016042667130629223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,1023,0.015829333414634068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,2047,0.027109332382678986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,2047,0.027263998985290527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,4095,0.034416000048319496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,8191,0.047135998805363975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,8191,0.06060799956321716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,16383,0.1069653332233429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,3,0.01897066707412402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,1,0.018965333700180054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,3,0.019258666783571243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,16383,0.08297066887219746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,1,0.019445333629846573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,32767,0.1900320053100586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,32767,0.14357866843541464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,7,0.019978666057189304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,7,0.01974933346112569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,15,0.020314666132132213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,15,0.020469332734743755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,31,0.02475200096766154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,31,0.025450666745503742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,63,0.025008000433444977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,63,0.02517866591612498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,127,0.02586666742960612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,127,0.0258240004380544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,255,0.03745066622893015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,255,0.030661332110563915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,511,0.060080001751581825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,511,0.05120000243186951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,1023,0.1411626636981964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,1023,0.08401599526405334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,2047,0.2709493239720662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,2047,0.17176000277201334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,1,0.007877333089709282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,1,0.009141333401203156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,3,0.007818666597207388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,4095,0.528218666712443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,15,0.008026666939258575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,4095,0.32561065753300983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,3,0.009205333267649015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,7,0.008293333152929941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,7,0.009103999783595404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,15,0.009103999783595404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,31,0.008122666428486506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,31,0.009445333232482275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,63,0.008767999708652496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,63,0.010005333150426546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,127,0.009114666531483332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,127,0.011509332805871964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,255,0.00922133338948091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,255,0.011551999797423681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,511,0.0102186668664217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,511,0.011546666423479715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,1023,0.02613866577545802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,1023,0.025909334421157837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,2047,0.030426666140556335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,2047,0.030005333324273426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,4095,0.04675200084845225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,8191,0.08777067065238953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,4095,0.04027733455101649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,8191,0.06414933502674103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,16383,0.15918933351834616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,16383,0.11431466539700826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,1,0.03427733232577642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,3,0.03389866650104523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,1,0.03365866591533025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,3,0.03542399903138479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,32767,0.20983999967575073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,7,0.03493333359559377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,32767,0.2909333308537801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,7,0.035487999518712364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,15,0.03617066641648611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,15,0.03706666578849157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,31,0.044938668608665466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,31,0.04596266647179922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,63,0.045519997676213585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,63,0.04593066871166229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,127,0.05197866757710775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,127,0.04624533156553904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,255,0.06640000144640605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,255,0.06233599781990051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,511,0.10815466443697612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,511,0.09501866499582927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,1023,0.27055466175079346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,1023,0.15813333789507547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,2047,0.5296106735865275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,1,0.06251200040181477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,2047,0.32841600974400836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,1,0.06403733293215434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,3,0.0625493327776591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,3,0.06409066418806712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,7,0.06487999856472015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,7,0.06608533362547557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,15,0.06668266654014587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,15,0.0682666649421056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,31,0.08456533153851827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,31,0.08559999863306682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,63,0.09246933460235596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,63,0.08678932984670003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,127,0.09567466378211975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,127,0.09408533573150635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,255,0.12081600228945415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,255,0.11637866497039795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,511,0.2003893256187439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,511,0.17893866697947183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,float16,1,0.11878933509190877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,fp8,1,0.12186132868131001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,float16,3,0.11929066975911458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,fp8,3,0.12098133563995361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,float16,7,0.1228053371111552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,fp8,7,0.12693867087364197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,float16,63,0.1775146722793579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,float16,15,0.13269333044687906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,fp8,15,0.13144000371297201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,float16,31,0.17147199312845865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,fp8,31,0.1695093313852946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,fp8,63,0.1774346629778544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,1,0.008832000195980072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,float16,127,0.181002676486969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,fp8,127,0.18140800793965658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,float16,255,0.23100266853968301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,1,0.008703999842206636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,fp8,255,0.22257065773010254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,31,0.009488000224033991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,3,0.008821333448092142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,7,0.009008000294367472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,3,0.008853333070874214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,7,0.008832000195980072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,15,0.008789333204428354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,127,0.011312000453472137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,15,0.009002666920423508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,31,0.009493333597977957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,63,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,63,0.012693333129088083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,127,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,255,0.011391999820868174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,255,0.011285333583752314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,511,0.013376000026861826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,511,0.012805332740147909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,1023,0.018266666680574417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,1023,0.01844800015290578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,2047,0.03898133337497711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,2047,0.028778667251269024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,4095,0.07627200086911519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,4095,0.0572213331858317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,8191,0.10473066568374634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,8191,0.14121066530545553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,16383,0.27702399094899494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,16383,0.19912532965342203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,32767,0.6805386543273926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,32767,0.581712007522583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,float16,1,0.231605331103007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,fp8,1,0.237664004166921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,float16,3,0.23189866542816162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,fp8,3,0.23790399233500162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,float16,7,0.25727999210357666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,fp8,7,0.24810133377710977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,float16,15,0.27537065744400024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,fp8,15,0.2731413245201111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,float16,31,0.33535468578338623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,fp8,31,0.33767998218536377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,float16,63,0.346346656481425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,fp8,63,0.34906665484110516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,float16,127,0.3524266481399536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,fp8,127,0.3522719939549764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,float16,1,0.49054400126139325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,fp8,1,0.47808531920115155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,float16,3,0.5041759808858236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,fp8,3,0.5010826587677002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,float16,7,0.5217386484146118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,fp8,7,0.5270506540934244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,float16,15,0.5437440077463785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,1,0.013781332721312841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,1,0.013754667093356451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,3,0.013717333475748697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,3,0.013898666948080063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,7,0.013850666582584381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,7,0.013967999567588171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,15,0.013999999811251959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,fp8,15,0.5553013483683268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,float16,31,0.6633919874827067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,15,0.013973332941532135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,31,0.013733333597580591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,31,0.013845333208640417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,fp8,31,0.6685653527577718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,63,0.015722667177518208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,float16,63,0.6841973463694254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,511,0.01923199991385142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,63,0.015599999576807022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,fp8,63,0.6904533704121908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,127,0.01570133368174235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,127,0.015647999942302704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,255,0.016149333367745083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,255,0.015626666446526844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,511,0.01851733277241389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,1023,0.03399466723203659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,1023,0.026367999613285065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,2047,0.04818666477998098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,2047,0.07938133180141449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,4095,0.15123200416564941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,4095,0.09380799531936646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,8191,0.2868000070254008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,8191,0.18487467368443808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,1,0.012709333250919977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,1,0.01313599944114685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,16383,0.566431999206543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,7,0.013338666409254074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,16383,0.3572479883829753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,3,0.01302933320403099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,3,0.013338666409254074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,7,0.01423466702302297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,15,0.014837333311637243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,15,0.01509333277742068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,31,0.015216000378131866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,31,0.015194666882356008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,63,0.01515199989080429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,63,0.01524266724785169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,127,0.01770666614174843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,1023,0.15005333224932352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,127,0.01770666614174843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,255,0.03385599950949351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,255,0.0262719988822937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,511,0.0773226668437322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,511,0.048453330993652344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,1023,0.10046933094660442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,2047,0.19261866807937622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,2047,0.28726933399836224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,4095,0.5596799850463867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,4095,0.36554133892059326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,1,0.008757333581646284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,1,0.008842666943868002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,3,0.008629333227872849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,3,0.008805333326260248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,7,0.0086666668454806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,8191,0.7421387036641439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,8191,1.1288053194681804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,7,0.008832000195980072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,15,0.008842666943868002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,15,0.00891733355820179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,31,0.009503999724984169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,31,0.009749333063761393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,63,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,63,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,127,0.011333333949247995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,127,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,255,0.011349332829316458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,255,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,511,0.015381333728631338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,511,0.015573333948850632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,1023,0.016575999557971954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,1023,0.01669866715868314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,2047,0.016810666769742966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,2047,0.016554666062196095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,4095,0.034645333886146545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,4095,0.034186666210492454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,8191,0.05947199960549673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,8191,0.047210668524106346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,16383,0.10779199997584026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,16383,0.0842133363087972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,1,0.00884799969693025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,1,0.00891733355820179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,32767,0.19182399908701578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,3,0.008890666688481966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,32767,0.14985066652297974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,3,0.009103999783595404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,7,0.008826666822036108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,7,0.008976000050703684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,15,0.009018666421373686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,15,0.009082666908701261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,31,0.009722666814923286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,31,0.009679999823371569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,63,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,63,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,127,0.011349332829316458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,127,0.011472000430027643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,255,0.01146666705608368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,1023,0.026863999664783478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,255,0.011503999431928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,511,0.015770666301250458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,511,0.015834666788578033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,1023,0.02739733209212621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,2047,0.03480533262093862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,2047,0.0340693344672521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,4095,0.06079466640949249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,4095,0.04738133152325948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,8191,0.1067626674969991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,8191,0.08353599905967712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,16383,0.15018133322397867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,16383,0.19166932503382364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,1,0.018640000373125076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,32767,0.3641386826833089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,1,0.019146667172511418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,32767,0.2762613296508789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,3,0.019152000546455383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,3,0.019893333315849304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,7,0.02004266654451688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,15,0.02476266771554947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,7,0.02027733375628789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,15,0.026629333694775898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,31,0.025093334416548412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,31,0.02510400116443634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,63,0.025429333249727886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,63,0.025274666647116344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,127,0.037818667789300285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,127,0.030405332644780476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,255,0.06003733476003011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,2047,0.32653866211573285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,255,0.051114668448766075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,511,0.14240533113479614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,511,0.08479467034339905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,1023,0.2707306742668152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,1023,0.17222400506337485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,2047,0.526042660077413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,1,0.00902399979531765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,7,0.009178666397929192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,1,0.009408000235756239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,15,0.009183999771873156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,4095,0.6347999970118204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,31,0.009999999776482582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,3,0.008885333314538002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,3,0.00921066664159298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,7,0.009183999771873156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,4095,1.0397706826527913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,15,0.009317333499590555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,31,0.00997866690158844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,63,0.011701333026091257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,63,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,127,0.011535999675591787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,127,0.011418666690587997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,2047,0.04646400113900503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,255,0.012191999703645706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,511,0.026149332523345947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,255,0.011749333391586939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,511,0.02587733417749405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,1023,0.030080000559488933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,1023,0.02962133288383484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,2047,0.04049599915742874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,4095,0.0874079962571462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,4095,0.06611200173695882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,32767,0.5590293407440186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,8191,0.1560533344745636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,8191,0.11956800023714702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,16383,0.2905919949213664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,16383,0.21986132860183716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,32767,0.4192213217417399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,1,0.03346666693687439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,1,0.034559999903043113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,3,0.034688000877698265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,3,0.03606933355331421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,7,0.03621866554021835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,7,0.03666666646798452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,15,0.04545066754023234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,15,0.04558399816354116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,31,0.04585599899291992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,255,0.10852799812952678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,31,0.045381332437197365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,63,0.05234666665395101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,63,0.04651199777921041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,127,0.06646400193373363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,127,0.06284266710281372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,255,0.09521599610646565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,511,0.2712159951527913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,511,0.1595306694507599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,1023,0.3306826750437419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,1,0.062261333068211876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,1023,0.5294186671574911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,1,0.06422399977842967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,2047,0.6375466585159302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,3,0.06414933502674103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,2047,1.0391466617584229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,3,0.06600533425807953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,7,0.06716800232728322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,63,0.0942133367061615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,7,0.06894933183987935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,15,0.08598933617273967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,15,0.08559466401735942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,31,0.0879253347714742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,31,0.09270933270454407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,63,0.09624532858530681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,127,0.12355732917785645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,127,0.11660800377527873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,255,0.20409599939982095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,3,0.12435199817021687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,255,0.17932265996932983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,511,0.5285120010375977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,511,0.3049013415972392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,1,0.11732266346613567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,fp8,1,0.12191999951998393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,fp8,3,0.12652800480524698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,7,0.13826666275660196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,fp8,7,0.1333440045515696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,15,0.1737920045852661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,fp8,15,0.17270400126775107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,31,0.17971734205881754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,fp8,31,0.1779359976450602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,63,0.1834719975789388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,fp8,63,0.18061333894729614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,127,0.23388266563415527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,fp8,127,0.2262880007425944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,255,0.3875360091527303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,1,0.008506666868925095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,1,0.008672000219424566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,3,0.00895999992887179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,3,0.008874666566650072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,31,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,7,0.008778666456540426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,fp8,255,0.3503093322118123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,7,0.009050666665037474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,15,0.00927466650803884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,15,0.009530666594703993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,31,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,63,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,63,0.011349332829316458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,127,0.011530666301647821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,127,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,255,0.013023999830087027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,255,0.012975999464591345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,2047,0.053599998354911804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,511,0.018112000077962875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,511,0.018021332720915478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,1023,0.03311999887228012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,1023,0.028575999041398365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,2047,0.07549333572387695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,4095,0.14108799894650778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,4095,0.10393066207567851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,8191,0.275706668694814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,8191,0.19640533129374185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,16383,0.5462773243586222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,16383,0.3817066748936971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,32767,1.3533120155334473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,1,0.24987733364105225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,fp8,1,0.24235733350118002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,32767,1.1594453652699788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,fp8,3,0.26603732506434125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,3,0.266704003016154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,7,0.27750933170318604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,fp8,7,0.2861973245938619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,15,0.34233065446217853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,fp8,15,0.34590399265289307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,fp8,31,0.35133334000905353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,31,0.35177600383758545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,63,0.35780266920725506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,fp8,63,0.3531839847564697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,127,0.4547040065129598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,fp8,127,0.4437706470489502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,float16,1,0.5101600090662638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,fp8,1,0.5270400047302246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,float16,3,0.525546669960022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,fp8,3,0.5405386686325073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,float16,7,0.5483786662419637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,fp8,7,0.5671093463897705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,float16,15,0.677781343460083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,1,0.013354666531085968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,1,0.013882666826248169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,3,0.013093333691358566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,3,0.013605333864688873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,7,0.01320533330241839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,fp8,15,0.6848320166269938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,7,0.01369599997997284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,float16,31,0.6960906982421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,15,0.01369599997997284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,15,0.013797332843144735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,31,0.01563199982047081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,fp8,31,0.6984319686889648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,31,0.015610666324694952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,63,0.01586666703224182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,255,0.018730666488409042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,float16,63,0.7064692974090576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,fp8,63,0.6987360318501791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,63,0.015578666081031164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,127,0.016186666985352833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,1023,0.04956266780694326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,127,0.015872000406185787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,255,0.01921066641807556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,511,0.034416000048319496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,511,0.02683199942111969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,1023,0.07960000137488048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,2047,0.15028267105420431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,2047,0.09435199697812398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,4095,0.287445326646169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,4095,0.18564265966415405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,8191,0.5629813273747762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,1,0.013909333695967993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,8191,0.35886398951212567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,1,0.014218666901191076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,3,0.014090667168299357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,3,0.0141546664138635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,16383,0.7086559931437174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,31,0.015573333948850632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,31,0.015562667200962702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,63,0.018757333358128864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,7,0.014282666146755219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,16383,1.1184213161468506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,7,0.01431999976436297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,15,0.014474666366974512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,15,0.014469332993030548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,63,0.018677332748969395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,127,0.018805333723624546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,1023,0.044010668992996216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,127,0.018895999838908512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,255,0.019152000546455383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,255,0.019093333433071773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,2047,0.0983893374602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,511,0.02327466756105423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,511,0.022346665461858112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,1023,0.03401600072781245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,2047,0.059562668204307556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,4095,0.18481600284576416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,4095,0.11660800377527873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,8191,0.21624000867207846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,8191,0.3539359966913859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,1,0.007584000006318092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,7,0.009103999783595404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,1,0.009008000294367472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,3,0.007893333211541176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,16383,0.42053866386413574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,16383,0.7041599750518799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,3,0.008943999807039896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,7,0.007685333490371704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,15,0.007637333124876022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,15,0.009018666421373686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,31,0.008021333565314611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,31,0.00926399976015091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,63,0.008634666601816813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,127,0.008570666735370954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,63,0.010687999427318573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,127,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,255,0.008799999952316284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,255,0.011429333438475927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,511,0.009962666779756546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,511,0.011450666934251785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,1023,0.016048000504573185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,1023,0.015562667200962702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,4095,0.016943999876578648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,8191,0.027797333896160126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,2047,0.016672000288963318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,2047,0.016389333953460056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,4095,0.016965333372354507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,8191,0.02754133443037669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,16383,0.03841600070397059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,16383,0.03756800045569738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,32767,0.07758399844169617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,1,0.008000000069538752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,32767,0.05685333410898844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,1,0.00902399979531765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,3,0.008000000069538752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,7,0.007989333321650824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,3,0.009183999771873156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,7,0.008992000172535578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,15,0.007727999861041705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,15,0.009253333633144697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,31,0.00820266641676426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,65535,0.13590932885805765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,65535,0.10567466417948405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,31,0.009248000259200731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,63,0.00879466657837232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,63,0.009749333063761393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,127,0.009066666786869368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,127,0.011551999797423681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,255,0.008912000184257826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,255,0.011429333438475927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,511,0.00979200005531311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,511,0.011557333171367645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,1023,0.015749332805474598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,1023,0.015802666544914246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,2047,0.01720533271630605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,2047,0.016586666305859882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,4095,0.027973333994547527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,4095,0.027162666122118633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,8191,0.03884266565243403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,8191,0.03670933345953623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,16383,0.07502933343251546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,16383,0.05523733297983805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,32767,0.12462932864824931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,1,0.013818666338920593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,32767,0.09594133496284485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,1,0.014325333138306936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,3,0.014186666657527288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,3,0.014106666048367819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,65535,0.22523732980092367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,65535,0.1671839952468872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,7,0.014186666657527288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,7,0.01458666721979777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,15,0.015029333531856537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,15,0.015226667126019796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,31,0.017914666483799618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,31,0.019189332922299702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,63,0.018186666071414948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,255,0.021914665897687275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,63,0.018426666657129925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,127,0.018432000031073887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,127,0.018768000106016796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,255,0.02310933421055476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,511,0.042250668009122215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,511,0.033488000432650246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,1023,0.09830400347709656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,1023,0.05878399809201559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,1,0.007733333234985669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,2047,0.18568533658981323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,2047,0.11759466926256816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,4095,0.35464000701904297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,4095,0.2198293407758077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,1,0.009088000282645226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,3,0.007802666475375493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,8191,0.720906654993693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,3,0.009119999905427298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,8191,0.4388586680094401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,7,0.008069333309928576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,7,0.009269333134094873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,15,0.007946666950980822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,15,0.009072000160813332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,31,0.007967999825874964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,31,0.009290666629870733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,63,0.008799999952316284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,63,0.00984533317387104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,127,0.009018666421373686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,127,0.011567999919255575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,255,0.00901333304742972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,255,0.011498666057984034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,511,0.009877333417534828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,511,0.01156266654531161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,1023,0.01640533283352852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,1023,0.01590399940808614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,2047,0.02824000020821889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,2047,0.027514666318893433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,4095,0.03666666646798452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,4095,0.034527999659379326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,8191,0.06545599798361461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,8191,0.048112000028292336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,16383,0.08398933211962382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,16383,0.11385066310564677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,32767,0.1480959951877594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,32767,0.20393067598342896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,1,0.02388266722361247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,1,0.024383999407291412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,3,0.02402133246262868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,7,0.025120000044504803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,65535,0.38306132952372235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,3,0.024570666253566742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,65535,0.2724799911181132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,7,0.02477866659561793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,15,0.025621332228183746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,15,0.026144000391165417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,31,0.031328000128269196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,31,0.032298666735490165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,255,0.04027199993530909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,63,0.03204799940188726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,63,0.03329599897066752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,127,0.033717334270477295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,127,0.03269333392381668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,255,0.04748799900213877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,511,0.07639466722806294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,511,0.06569600105285645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,1023,0.18504534165064493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,1023,0.10955199599266052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,2047,0.22542399168014526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,2047,0.35732801755269367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,1,0.042954668402671814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,1,0.04428799947102865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,4095,0.4301226536432902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,3,0.043141335248947144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,4095,0.697925329208374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,3,0.04423999786376953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,7,0.04504533112049103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,7,0.04576000074545542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,15,0.04664533336957296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,15,0.04715733230113983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,31,0.05850133299827576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,31,0.059008002281188965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,63,0.059706668059031166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,63,0.0617439995209376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,127,0.06622399886449178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,127,0.0618399977684021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,255,0.08496532837549846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,1,0.08146666487058003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,255,0.07955733438332875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,511,0.13921067118644714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,511,0.12266133228937785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,1023,0.35732801755269367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,1023,0.20755199591318765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,7,0.08659733335177104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,1,0.08260266482830048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,3,0.08165333171685536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,3,0.0832479993502299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,63,0.12001066406567891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,7,0.08360532919565837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,15,0.08725333213806152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,15,0.08916800220807393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,31,0.11493866642316182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,31,0.11286933223406474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,63,0.11851200461387634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,127,0.12426666418711345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,1,0.009328000247478485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,127,0.12286399801572163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,3,0.009370666618148485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,255,0.15705600380897522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,255,0.15199466546376547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,511,0.26238399744033813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,1,0.008122666428486506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,511,0.23603200912475586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,3,0.008112000301480293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,7,0.007920000081261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,7,0.009338666374484697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,15,0.00789866658548514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,15,0.009237333511312803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,31,0.008133333176374435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,31,0.00955200009047985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,63,0.008821333448092142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,63,0.01027199998497963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,127,0.008992000172535578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,127,0.011674666156371435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,255,0.009119999905427298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,255,0.011594666788975397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,511,0.010255999863147736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,511,0.011941333611806234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,1023,0.027210667729377747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,1023,0.02621866762638092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,2047,0.03290133426586787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,2047,0.03052799900372823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,4095,0.053871999184290566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,4095,0.04091733445723852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,8191,0.10824533303578694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,8191,0.06608533362547557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,16383,0.1936053236325582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,16383,0.12533332904179892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,32767,0.36320531368255615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,32767,0.2251573403676351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,float16,1,0.15642666816711426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,fp8,1,0.16075199842453003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,float16,3,0.15685333808263144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,fp8,3,0.160863995552063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,65535,0.7172213395436605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,float16,7,0.16275200247764587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,65535,0.43350398540496826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,fp8,7,0.1673706571261088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,float16,15,0.18402133385340372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,fp8,15,0.17405333121617636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,float16,31,0.2262186606725057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,float16,63,0.23187732696533203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,fp8,31,0.22650132576624551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,fp8,63,0.23366934061050415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,float16,127,0.23931733767191568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,fp8,127,0.2391200065612793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,float16,255,0.303056001663208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,fp8,255,0.29689600070317584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,float16,1,0.30646934111913043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,fp8,1,0.3132373293240865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,float16,3,0.3214026689529419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,fp8,3,0.31670933961868286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,float16,7,0.3500906626383464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,fp8,7,0.3402613401412964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,float16,15,0.3672373294830322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,fp8,15,0.3696746826171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,float16,31,0.44137601057688397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,1,0.008592000231146812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,fp8,31,0.4482186635335286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,float16,63,0.45878398418426514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,fp8,63,0.4629173278808594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,1,0.008656000097592672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,3,0.008517333616813024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,float16,127,0.4700640042622884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,3,0.008629333227872849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,fp8,127,0.4707253376642863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,7,0.008992000172535578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,7,0.008858666444818178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,15,0.008821333448092142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,15,0.009029333169261614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,31,0.009392000113924345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,31,0.009541333342591921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,63,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,63,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,127,0.011407999942700068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,127,0.011312000453472137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,255,0.011445333560307821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,255,0.011482667177915573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,511,0.013232000172138214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,511,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,1023,0.01883200059334437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,1023,0.018181333939234417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,2047,0.05286400020122528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,2047,0.029050665597120922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,4095,0.0960106650988261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,4095,0.06169599791367849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,8191,0.18153067429860434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,8191,0.11202133695284526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,16383,0.35863999525705975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,16383,0.21644800901412964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,1,0.013770667215188345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,1,0.014293332894643148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,3,0.014090667168299357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,3,0.014389333625634512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,7,0.013882666826248169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,32767,0.6048640012741089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,32767,0.78875199953715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,7,0.01452800010641416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,15,0.015157333264748255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,15,0.015578666081031164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,31,0.018581333259741466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,31,0.0186666672428449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,63,0.01860800012946129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,63,0.018842666099468868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,127,0.019285333653291065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,127,0.019071999937295914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,255,0.023455999791622162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,255,0.02231466770172119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,511,0.043749332427978516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,511,0.03465600063403448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,1023,0.09829866886138916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,1023,0.05983999868233999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,2047,0.18403732776641846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,2047,0.11620266238848369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,4095,0.3538026809692383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,4095,0.2172693411509196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,8191,0.4198933442433675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,3,0.008890666688481966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,16383,0.8358293374379476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,7,0.00873066671192646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,8191,0.6964693069458008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,1,0.008933333059151968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,1,0.008943999807039896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,3,0.008826666822036108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,16383,1.3960960706075032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,7,0.008863999818762144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,15,0.009050666665037474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,127,0.011285333583752314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,15,0.00902399979531765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,31,0.009541333342591921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,31,0.009872000043590864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,63,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,63,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,127,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,255,0.011365332951148352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,255,0.01145600030819575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,511,0.01544533297419548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,511,0.015253332753976187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,1023,0.016682667036851246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,1023,0.016373333831628162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,2047,0.016773333152135212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,2047,0.01658133293191592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,4095,0.02756800005833308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,4095,0.0273333340883255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,8191,0.03792533278465271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,8191,0.037392000357309975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,16383,0.0584853341182073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,16383,0.07788266738255818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,32767,0.13782933354377747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,32767,0.10940266648928325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,1,0.009029333169261614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,1,0.008933333059151968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,3,0.008762666955590248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,3,0.009039999917149544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,7,0.008853333070874214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,7,0.009162666896979014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,65535,0.2513920068740845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,65535,0.1928053299585978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,15,0.009093333035707474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,15,0.009072000160813332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,31,0.009712000067035357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,31,0.009797333429257074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,63,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,63,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,127,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,127,0.011407999942700068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,255,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,255,0.011418666690587997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,511,0.015615999698638916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,511,0.015664000064134598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,1023,0.016751999656359356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,1023,0.016490666816631954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,2047,0.027434666951497395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,2047,0.027119999130566914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,16383,0.1258026659488678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,4095,0.03817066550254822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,4095,0.036992001036802925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,8191,0.07388266424338023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,8191,0.055231998364130654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,16383,0.09876267115275066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,32767,0.22697599728902182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,32767,0.17414933443069458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,1,0.013850666582584381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,65535,0.4331253369649251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,1,0.014074667046467463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,3,0.014111999422311783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,65535,0.33455999692281085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,3,0.014373333503802618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,7,0.014655999839305878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,7,0.014848000059525171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,15,0.017957333475351334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,15,0.01809599995613098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,31,0.01810666670401891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,31,0.01821333294113477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,63,0.01821333294113477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,63,0.018112000077962875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,127,0.02256533255179723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,127,0.02145066608985265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,255,0.04251733422279358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,255,0.03345600018898646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,511,0.09830400347709656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,511,0.05898133416970571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,1023,0.18381333351135254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,1023,0.11796266833941142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,2047,0.35465065638224286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,2047,0.2218666672706604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,1,0.008858666444818178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,4095,0.4297066529591878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,3,0.008992000172535578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,4095,0.6964960098266602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,7,0.008954666554927826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,1,0.008858666444818178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,3,0.008853333070874214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,7,0.008832000195980072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,8191,0.8633386294047037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,15,0.00915733352303505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,31,0.009685333197315535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,15,0.009258666386206945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,8191,1.4304426511128743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,31,0.009626666704813639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,511,0.016399999459584553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,63,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,63,0.011338666081428528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,127,0.011424000064531961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,127,0.01139733319481214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,255,0.011535999675591787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,255,0.011685332904259363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,511,0.015615999698638916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,1023,0.027813332776228588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,1023,0.027535999814669292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,2047,0.036101333796978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,2047,0.03437866767247518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,4095,0.06181333462397257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,4095,0.04825599988301595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,8191,0.1130400002002716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,8191,0.085999995470047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,16383,0.20307733615239462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,16383,0.1542026698589325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,32767,0.3819040060043335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,32767,0.28543466329574585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,1,0.02426133304834366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,1,0.02462399999300639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,65535,0.7411786715189616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,65535,0.547823985417684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,3,0.024421334266662598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,3,0.025237334271272022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,7,0.025290665527184803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,7,0.025765334566434223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,15,0.03186666717131933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,15,0.03199466566244761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,31,0.03209600100914637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,31,0.0317493329445521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,63,0.03396799912055334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,63,0.03201066702604294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,127,0.04775466521581014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,255,0.06625600159168243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,127,0.0412266676624616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,255,0.07646400233109792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,511,0.185589333375295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,511,0.11043733358383179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,2047,0.4339040120442708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,1023,0.35707199573516846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,1023,0.22665599981943765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,2047,0.6965013345082601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,1,0.04271466533342997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,1,0.04438933233420054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,3,0.04471466441949209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,3,0.04568533102671305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,4095,0.8447999954223633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,4095,1.3812586466471355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,7,0.04650133351484934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,7,0.04721599817276001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,15,0.058304001887639366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,15,0.058917333682378135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,31,0.06235733131567637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,31,0.05877333382765452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,63,0.06677333513895671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,63,0.06291733185450236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,511,0.20772800842920938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,127,0.08899733424186707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,127,0.08005333443482716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,255,0.1407360037167867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,255,0.1234879990418752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,511,0.3579626480738322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,1,0.08030933141708374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,1023,0.436352014541626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,1,0.08382399876912434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,3,0.08330666522185008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,1023,0.7022293408711752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,3,0.08617066343625386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,7,0.08798933029174805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,7,0.09044266740481059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,15,0.11649066209793091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,15,0.11281067132949829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,31,0.12249599893887837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,31,0.11899200081825256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,63,0.12307199835777283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,63,0.12517333030700684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,127,0.16029333074887595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,127,0.15357333421707153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,255,0.2645866672197978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,255,0.23712533712387085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,1,0.00892800030608972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,511,0.7010026772816976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,1,0.009226666763424873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,15,0.009653333574533463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,511,0.3997866710027059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,3,0.00915733352303505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,3,0.009429333110650381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,7,0.008879999940594038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,7,0.009423999736706415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,15,0.00933333362142245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,31,0.009882666791478792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,31,0.010341333225369453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,63,0.011493333925803503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,63,0.011695999652147293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,127,0.011727999895811081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,127,0.011525332927703857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,255,0.012319999436537424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,255,0.011727999895811081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,511,0.026687999566396076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,511,0.02611733227968216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,1023,0.030853333572546642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,1023,0.030421334008375805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,2047,0.05297600229581197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,2047,0.0415786678592364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,4095,0.10685333609580994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,16383,0.36177066961924237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,4095,0.0689279983441035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,8191,0.19174933433532715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,8191,0.13150933384895325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,16383,0.23745065927505493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,32767,0.7006133397420248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,32767,0.45045332113901776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,1,0.15635200341542563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,fp8,1,0.1604213317235311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,3,0.16967999935150146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,fp8,3,0.16940800348917642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,65535,0.8914399941762289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,7,0.1868106722831726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,fp8,7,0.1814240018526713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,65535,1.4028533299763997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,15,0.23040000597635904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,fp8,15,0.23337066173553467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,fp8,63,0.23868799209594727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,31,0.2369706630706787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,fp8,31,0.23533866802851358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,63,0.2395520011583964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,127,0.3136799931526184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,fp8,127,0.29891733328501385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,fp8,255,0.4609653155008952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,255,0.511733333269755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,1,0.34419198830922443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,fp8,1,0.3475573460261027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,3,0.350106676419576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,fp8,3,0.36246931552886963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,7,0.3702186743418376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,fp8,7,0.3777759869893392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,15,0.4538666804631551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,fp8,15,0.4617973168690999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,31,0.4631893237431844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,1,0.008602666358153025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,3,0.008672000219424566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,1,0.00874133345981439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,fp8,31,0.46278401215871173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,3,0.008863999818762144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,7,0.008703999842206636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,63,0.473962664604187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,fp8,63,0.4718293348948161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,15,0.00961599995692571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,7,0.008826666822036108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,15,0.00926399976015091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,127,0.5985066493352255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,fp8,127,0.5896106561024984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,31,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,31,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,63,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,63,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,127,0.011328000575304031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,127,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,255,0.013295999417702356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,511,0.018719999740521114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,255,0.013264000415802002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,511,0.0183146670460701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,1023,0.05221866567929586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,1023,0.028629332780838013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,4095,0.11268267035484314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,2047,0.09516800443331401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,2047,0.06222933530807495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,4095,0.18079467614491782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,8191,0.21422400077184042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,8191,0.35443198680877686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,1,0.013834666460752487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,16383,0.42692800362904865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,16383,0.7092320124308268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,1,0.013872000078360239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,3,0.014032000054915747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,3,0.014053333550691605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,7,0.014042666802803675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,31,0.013839999834696451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,32767,1.5499253273010254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,7,0.014346666634082794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,32767,1.2221279939015706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,15,0.013973332941532135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,15,0.014032000054915747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,31,0.013994666437307993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,255,0.015658666690190632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,63,0.01534933348496755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,63,0.01563199982047081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,127,0.015754666179418564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,127,0.015781333049138386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,255,0.016106666376193363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,511,0.018837332725524902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,511,0.01852799952030182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,1023,0.034416000048319496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,1023,0.025946666797002155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,2047,0.08024533092975616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,2047,0.04864533245563507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,4095,0.15130133430163065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,1,0.007776000226537387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,4095,0.09425600369771321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,16383,0.5667786598205566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,8191,0.28841066360473633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,8191,0.18501333395640054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,7,0.008842666943868002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,1,0.008992000172535578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,16383,0.35547733306884766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,3,0.007802666475375493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,31,0.009216000015536943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,3,0.009109333157539368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,7,0.007536000261704127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,127,0.011343999455372492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,15,0.007610666876037915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,15,0.009152000149091085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,31,0.007850666840871176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,63,0.008714666590094566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,63,0.00972800018886725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,127,0.008858666444818178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,1023,0.015471999843915304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,255,0.00871999996403853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,255,0.011322667201360067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,511,0.009973333527644476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,511,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,4095,0.016522667060295742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,1023,0.015658666690190632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,2047,0.016832000265518825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,2047,0.01621866722901662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,4095,0.016666666915019352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,8191,0.017312000195185345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,8191,0.016704000532627106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,16383,0.02096533278624217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,16383,0.020400000115235645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,32767,0.04088533421357473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,32767,0.026650667190551758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,1,0.007711999739209811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,1,0.008826666822036108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,3,0.007765333478649457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,3,0.009258666386206945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,65535,0.07885866860548656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,7,0.007685333490371704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,7,0.008853333070874214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,15,0.007621333623925845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,15,0.008938666433095932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,65535,0.05291733145713806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,31,0.008218666538596153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,31,0.009045333291093508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,63,0.008618666479984919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,63,0.00972800018886725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,127,0.008799999952316284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,127,0.011391999820868174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,255,0.00901333304742972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,255,0.011567999919255575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,511,0.009813333551088968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,511,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,1023,0.01565333331624667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,1023,0.015594666202863058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,2047,0.016730666160583496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,2047,0.016629333297411602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,4095,0.01728533332546552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,32767,0.10637866457303365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,4095,0.01682666689157486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,8191,0.035071998834609985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,8191,0.03416533271471659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,16383,0.061887999375661217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,16383,0.04714666803677877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,32767,0.08353599905967712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,1,0.013733333597580591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,1,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,3,0.013173333058754602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,65535,0.14366400241851807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,3,0.013248000293970108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,65535,0.1897653341293335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,7,0.013210666676362356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,7,0.013605333864688873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,15,0.013642666240533194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,15,0.013183999806642532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,31,0.014896000425020853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,31,0.015189333508412043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,63,0.015200000256299973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,511,0.03418133407831192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,63,0.015402667224407196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,127,0.01532799998919169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,127,0.015562667200962702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,255,0.01821333294113477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,255,0.017690667261679966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,511,0.025573333104451496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,1023,0.07881066699822743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,1023,0.04758933186531067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,2047,0.15063466628392538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,2047,0.09858133395512898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,4095,0.2879093289375305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,4095,0.1875306765238444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,8191,0.5739946762720743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,1,0.007882666463653246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,8191,0.3816800117492676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,1,0.00903466654320558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,3,0.00797333319981893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,3,0.009082666908701261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,7,0.0080960001796484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,63,0.008869333192706108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,7,0.009381333366036415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,127,0.009152000149091085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,15,0.007861333588759104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,15,0.009109333157539368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,31,0.007962666451931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,31,0.00926399976015091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,63,0.009872000043590864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,127,0.011472000430027643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,255,0.009194666519761086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,255,0.011674666156371435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,511,0.009952000031868616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,511,0.011541333049535751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,1023,0.01587733378012975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,1023,0.01578666642308235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,2047,0.027258666853109997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,2047,0.027189334233601887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,4095,0.03522666543722153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,4095,0.03465066601832708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,8191,0.05929600199063619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,8191,0.04684799909591675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,16383,0.105103999376297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,16383,0.08310399949550629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,32767,0.1898720065752665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,1,0.018735999862353008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,32767,0.1444480021794637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,1,0.019205333044131596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,3,0.018794666975736618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,3,0.019333332777023315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,7,0.019391999890406925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,65535,0.3604373137156169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,65535,0.2661706606547038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,7,0.02011200040578842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,15,0.02027200038234393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,15,0.0204373337328434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,31,0.02483733246723811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,31,0.025087999800841015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,63,0.025360000630219776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,63,0.025578667720158894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,127,0.025781333446502686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,127,0.02573866645495097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,255,0.03777066618204117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,255,0.031109333038330078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,511,0.05952000121275584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,2047,0.2716640035311381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,511,0.0513919989267985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,1023,0.14261866609255472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,1023,0.08438400427500407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,2047,0.17122666041056314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,1,0.03363200028737386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,1,0.03457066665093104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,4095,0.5288853247960409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,4095,0.3242826660474141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,3,0.034074666599432625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,3,0.034389334420363106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,7,0.03497066597143809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,7,0.03566933423280716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,15,0.03606933355331421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,63,0.045834665497144066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,15,0.037087999284267426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,31,0.04510400195916494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,31,0.04542933404445648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,63,0.04619733492533366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,127,0.05207466582457224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,127,0.046495998899141945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,255,0.06611200173695882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,255,0.062421331803003945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,511,0.10822932918866475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,511,0.09500267108281453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,1023,0.27108800411224365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,1023,0.1585599978764852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,1,0.06253333389759064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,1,0.06358933448791504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,3,0.06225066880385081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,15,0.06850666801134746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,3,0.06384000182151794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,7,0.06477866570154826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,7,0.06651733318964641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,15,0.06644266843795776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,31,0.08521067102750142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,31,0.085125337044398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,63,0.09211732943852742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,63,0.08662399649620056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,127,0.09578667084376018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,127,0.0946666697661082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,1,0.009066666786869368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,255,0.12165333827336629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,255,0.11521599690119426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,511,0.2018079956372579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,511,0.17774933576583862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,1,0.007877333089709282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,15,0.007840000092983246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,3,0.008005333443482717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,3,0.009226666763424873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,7,0.007914666707317034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,7,0.00933333362142245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,15,0.00919999989370505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,31,0.008181333541870117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,31,0.009285333255926767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,63,0.008703999842206636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,63,0.009946666657924652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,127,0.009098666409651438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,127,0.011472000430027643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,1023,0.025765334566434223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,255,0.009402666861812273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,255,0.011642667154471079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,511,0.010314666976531347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,511,0.011600000162919363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,1023,0.026015999416510265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,2047,0.030458666384220123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,2047,0.030282666285832722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,4095,0.04673066735267639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,4095,0.04043733328580856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,8191,0.08690133690834045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,32767,0.2911413311958313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,32767,0.210533340771993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,8191,0.06407466530799866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,16383,0.1570186714331309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,16383,0.11398399869600932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,float16,1,0.11910933256149292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,fp8,1,0.12171199917793274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,float16,3,0.11829333504041036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,65535,0.39903998374938965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,65535,0.5707093477249146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,fp8,3,0.1209440032641093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,float16,7,0.12268267075220744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,fp8,7,0.1269439955552419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,float16,15,0.13358933726946512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,fp8,15,0.13056000073750815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,float16,31,0.17132800817489624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,fp8,31,0.16851200660069784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,float16,63,0.17598400513331094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,fp8,63,0.17701866229375204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,float16,127,0.18178667624791464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,fp8,127,0.18134933710098267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,float16,255,0.22991466522216797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,fp8,255,0.2238453427950541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,fp8,1,0.23622934023539224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,float16,1,0.23261332511901855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,float16,3,0.23241066932678223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,fp8,3,0.23819732666015625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,float16,7,0.2566133340199788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,float16,15,0.27740800380706787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,fp8,15,0.2716053326924642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,fp8,31,0.3378080129623413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,float16,63,0.3461173375447591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,fp8,7,0.2480213244756063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,1,0.00867733359336853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,1,0.008687999720374743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,fp8,63,0.34913599491119385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,3,0.008837333569924036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,3,0.008693333094318708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,float16,31,0.3353866736094157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,7,0.008885333314538002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,7,0.00877333308259646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,float16,127,0.3548266490300496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,15,0.008767999708652496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,15,0.009082666908701261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,31,0.009557333464423815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,fp8,127,0.35225598017374676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,63,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,31,0.00943999985853831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,511,0.013082666943470636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,63,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,127,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,127,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,255,0.011637333780527115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,255,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,511,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,1023,0.018191999445358913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,1023,0.018133333573738735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,2047,0.03621866554021835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,2047,0.028901333610216778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,4095,0.07609599828720093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,4095,0.05542399982611338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,8191,0.142767995595932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,8191,0.10457066694895427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,16383,0.2797706723213196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,16383,0.1974239945411682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,1,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,1,0.013455999394257864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,32767,0.6841653188069662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,3,0.013050666699806849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,3,0.013327999661366144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,32767,0.5877386728922526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,7,0.012960000584522883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,7,0.01320533330241839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,15,0.01360000049074491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,15,0.014010666559139887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,31,0.015413332730531693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,31,0.015605332950750986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,63,0.015824000040690105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,63,0.015882667154073715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,127,0.016037333756685257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,127,0.015872000406185787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,255,0.01886933296918869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,255,0.018960000326236088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,511,0.03475733349720637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,511,0.02651199946800868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,1023,0.07981866598129272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,4095,0.18731733163197836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,1023,0.04972266654173533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,2047,0.15158399939537048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,2047,0.09447466333707173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,4095,0.28814399242401123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,8191,0.358298659324646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,8191,0.5638773441314697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,1,0.008805333326260248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,1,0.008816000074148178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,3,0.008645333349704742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,3,0.008725333337982496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,16383,0.7093386650085449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,16383,1.1190826892852783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,7,0.008698666468262672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,7,0.00877333308259646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,15,0.009077333534757296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,15,0.008901333436369896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,31,0.009503999724984169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,31,0.009653333574533463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,63,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,63,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,127,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,127,0.011429333438475927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,255,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,255,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,511,0.015423999478419622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,511,0.015824000040690105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,1023,0.01632000009218852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,1023,0.016229332735141117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,2047,0.01655999943614006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,2047,0.01642666632930438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,4095,0.016677333662907284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,4095,0.01669866715868314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,8191,0.021290667355060577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,8191,0.020096000283956528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,16383,0.03975466638803482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,16383,0.027258666853109997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,32767,0.0784853349129359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,32767,0.05305600166320801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,1,0.008576000109314919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,1,0.008933333059151968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,65535,0.14291200041770935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,3,0.008885333314538002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,65535,0.09448533256848653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,3,0.008762666955590248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,7,0.008645333349704742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,7,0.00884799969693025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,15,0.009103999783595404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,15,0.009066666786869368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,31,0.009568000212311745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,31,0.009952000031868616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,63,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,63,0.011322667201360067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,127,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,127,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,255,0.01157333329319954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,255,0.011407999942700068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,511,0.0161013330022494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,511,0.015765332927306492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,1023,0.016666666915019352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,1023,0.016597333053747814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,2047,0.016917333006858826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,2047,0.01655999943614006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,16383,0.10777067144711812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,4095,0.03508266558249792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,4095,0.034202667574087776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,8191,0.060378665725390114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,8191,0.047594666481018066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,16383,0.08659733335177104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,65535,0.36258665720621747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,32767,0.19107733170191446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,32767,0.14973333477973938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,1,0.012928000340859095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,1,0.01302933320403099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,65535,0.2786666750907898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,3,0.01328533391157786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,3,0.013455999394257864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,7,0.013125333935022354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,7,0.013237333546082178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,15,0.014746667196353277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,15,0.015157333264748255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,63,0.015295999745527903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,31,0.015034666905800501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,31,0.015306666493415833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,63,0.015402667224407196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,127,0.018133333573738735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,127,0.01764800027012825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,255,0.03404266635576884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,255,0.025642665723959606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,511,0.07794666786988576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,511,0.04897066454092661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,1023,0.1497599979241689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,4095,0.5609973271687826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,1023,0.10048533479372661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,2047,0.2882399956385295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,2047,0.19121599197387695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,4095,0.3699253400166829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,1,0.008901333436369896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,1,0.008943999807039896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,3,0.00892800030608972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,3,0.009093333035707474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,8191,0.7434826691945394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,8191,1.1341173648834229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,7,0.008874666566650072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,7,0.009029333169261614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,15,0.009226666763424873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,15,0.009045333291093508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,31,0.009712000067035357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,31,0.009888000165422758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,63,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,127,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,63,0.011407999942700068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,127,0.01146666705608368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,255,0.011802667131026586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,255,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,511,0.015813333292802174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,2047,0.03439466655254364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,4095,0.059978668888409935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,511,0.01581866666674614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,1023,0.02720533311367035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,1023,0.027093333502610523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,2047,0.03477866699298223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,4095,0.047237331668535866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,8191,0.1069493293762207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,8191,0.08403733372688293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,16383,0.19169066349665323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,16383,0.15005333224932352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,32767,0.36051734288533527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,1,0.01859733338157336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,32767,0.27667200565338135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,1,0.019306667149066925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,65535,0.7036800384521484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,65535,0.5324693520863851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,3,0.019306667149066925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,3,0.019845332950353622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,7,0.01993600030740102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,7,0.020479999482631683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,15,0.024959998826185863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,15,0.025199999411900837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,31,0.025226667523384094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,31,0.02496533344189326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,255,0.05978666742642721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,63,0.02571200082699458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,63,0.025285333395004272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,127,0.037952000896135964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,127,0.03089066594839096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,255,0.0513919989267985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,511,0.08449066678682964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,511,0.14243732889493307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,1023,0.27155200640360516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,1023,0.17197332779566446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,2047,0.5274773438771566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,2047,0.3288639982541402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,1,0.03342933456103007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,4095,1.043434699376424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,4095,0.6393173138300577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,1,0.03454933315515518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,3,0.03489066660404205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,3,0.03576533248027166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,7,0.03581333408753077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,7,0.03696533292531967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,15,0.04509866734345754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,15,0.04577066500981649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,31,0.046112000942230225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,31,0.04558399816354116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,63,0.052527998884518944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,63,0.04618666569391886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,127,0.06650666892528534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,127,0.06277333199977875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,255,0.10884267091751099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,255,0.09552533427874248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,511,0.2723093430201213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,3,0.06380799909432729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,511,0.16054399808247885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,1023,0.5309919913609823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,1023,0.33125332991282147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,1,0.06185600161552429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,1,0.06419733166694641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,3,0.06656533479690552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,7,0.06701333324114482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,7,0.06835733354091644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,63,0.09504000345865886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,15,0.08646933237711589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,15,0.0857919951279958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,31,0.09286933143933614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,31,0.08748799562454224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,63,0.09602133433024089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,127,0.12293333808581035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,127,0.11674666404724121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,3,0.009103999783595404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,255,0.20345600446065268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,3,0.009008000294367472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,255,0.18010133504867554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,1,0.008901333436369896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,511,0.5279680093129476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,511,0.30322132507960003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,1,0.00903466654320558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,7,0.008933333059151968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,7,0.00927466650803884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,15,0.009098666409651438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,127,0.011717333147923151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,15,0.009205333267649015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,31,0.009775999933481216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,31,0.009930666536092758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,63,0.011493333925803503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,63,0.011402666568756104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,127,0.011674666156371435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,255,0.01190399999419848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,255,0.011871999750534693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,511,0.02590399980545044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,511,0.02571733295917511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,1023,0.030063999195893604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,1023,0.02972800036271413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,2047,0.046037331223487854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,2047,0.0403413325548172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,4095,0.08635200063387553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,4095,0.06634666522343953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,8191,0.15664000312487283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,8191,0.11937066912651062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,16383,0.2904319961865743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,16383,0.2215893268585205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,32767,0.557642658551534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,32767,0.41787199179331463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,1,0.11804266770680745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,fp8,1,0.12202133735020955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,3,0.12386666735013326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,fp8,3,0.1275200049082438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,7,0.13725866874059042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,65535,1.1061546802520752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,65535,0.821839968363444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,fp8,7,0.13218133648236594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,15,0.17468265692392984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,fp8,15,0.17253865798314413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,31,0.17959467569986978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,fp8,31,0.17809067169825235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,63,0.18370666106541952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,fp8,63,0.18184532721837363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,fp8,255,0.35040533542633057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,127,0.23256532351175943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,fp8,127,0.2262293299039205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,255,0.3883253335952759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,1,0.24921600023905435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,fp8,1,0.24682666858037314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,3,0.26654932896296185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,fp8,3,0.26665065685908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,7,0.2778880000114441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,fp8,7,0.2868480086326599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,15,0.3421386480331421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,fp8,15,0.3455573320388794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,31,0.3516746759414673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,fp8,31,0.3513866662979126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,63,0.3577440182367961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,1,0.008474666625261307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,3,0.008805333326260248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,fp8,63,0.3532533248265584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,1,0.008672000219424566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,127,0.4517600138982137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,3,0.009152000149091085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,fp8,127,0.44405333201090497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,7,0.008703999842206636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,7,0.00878399983048439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,15,0.009290666629870733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,15,0.009445333232482275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,31,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,31,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,63,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,63,0.011365332951148352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,511,0.018053332964579265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,127,0.011365332951148352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,127,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,255,0.012815999488035837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,255,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,511,0.01802666609485944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,1023,0.03219733387231827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,1023,0.02861333390076955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,2047,0.07526400188604991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,2047,0.05460800230503082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,4095,0.14272000392278036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,4095,0.10360532999038696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,8191,0.27741867303848267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,8191,0.19594667355219522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,16383,0.5498346487681071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,16383,0.38389865557352704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,1,0.0085333331177632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,1,0.008613333106040955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,3,0.008586666857202848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,3,0.008661333471536636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,15,0.008736000085870424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,31,0.009434666484594345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,7,0.008837333569924036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,32767,1.3537386258443196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,31,0.009535999968647957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,7,0.008938666433095932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,32767,1.162175973256429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,15,0.008687999720374743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,63,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,63,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,127,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,127,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,255,0.011488000551859537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,255,0.011349332829316458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,511,0.013354666531085968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,511,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,1023,0.018496000518401463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,1023,0.018272000054518383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,2047,0.05285866558551788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,2047,0.0288426677385966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,4095,0.09640000263849895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,4095,0.061850666999816895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,8191,0.1828213334083557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,8191,0.11236266295115153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,16383,0.35814932982126874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,16383,0.21596799294153848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,1,0.007621333623925845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,32767,0.7819413344065348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,32767,0.6049013137817383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,1,0.009072000160813332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,3,0.007861333588759104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,3,0.009930666536092758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,15,0.008922666932145754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,7,0.007637333124876022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,63,0.009717333440979322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,7,0.008858666444818178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,15,0.00785600021481514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,31,0.008010666817426682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,31,0.009296000003814697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,63,0.008618666479984919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,127,0.008746666833758354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,127,0.011338666081428528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,255,0.00878399983048439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,255,0.011488000551859537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,511,0.009941333283980688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,511,0.011535999675591787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,1023,0.015658666690190632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,1023,0.015450666348139444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,2047,0.016602666427691776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,2047,0.016341333587964375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,4095,0.01666133354107539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,4095,0.016575999557971954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,8191,0.01714133347074191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,8191,0.016773333152135212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,16383,0.02777066578467687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,16383,0.02720533311367035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,32767,0.038149334490299225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,65535,0.0792799989382426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,32767,0.03734400123357773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,1,0.007743999982873599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,65535,0.05788266658782959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,1,0.00892800030608972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,3,0.007818666597207388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,3,0.009098666409651438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,7,0.007781333600481351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,7,0.00914666677514712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,15,0.007701333612203598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,15,0.00891733355820179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,31,0.007962666451931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,31,0.009216000015536943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,131071,0.1409173309803009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,63,0.008682666967312494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,255,0.008890666688481966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,131071,0.10911466677983601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,63,0.009962666779756546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,127,0.008805333326260248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,127,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,255,0.011477333803971609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,511,0.011461333682139715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,4095,0.016602666427691776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,511,0.009824000298976898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,1023,0.01579733317097028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,1023,0.01568000018596649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,2047,0.016869333883126576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,2047,0.016447999825080235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,4095,0.016741332908471424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,8191,0.027610667049884796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,8191,0.027477333943049114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,16383,0.038405333956082664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,16383,0.03737066686153412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,32767,0.07776000102361043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,32767,0.05709866682688395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,1,0.014149333039919535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,3,0.013951999445756277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,65535,0.1349066694577535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,3,0.014325333138306936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,1,0.014096000542243322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,65535,0.10605333248774211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,7,0.014373333503802618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,7,0.014501333236694336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,15,0.014250667144854864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,15,0.014645333091417948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,31,0.015466666469971338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,31,0.015573333948850632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,63,0.01877333347996076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,63,0.018837332725524902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,131071,0.24865599473317465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,131071,0.1863306760787964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,127,0.018768000106016796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,127,0.018992000569899876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,255,0.018911999960740406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,511,0.023445333043734234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,255,0.019093333433071773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,511,0.022282667458057404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,1023,0.04359466830889384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,1023,0.034117333590984344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,2047,0.09826133648554485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,2047,0.05963733295599619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,4095,0.18532266219456991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,4095,0.11665067076683044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,8191,0.35602132479349774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,8191,0.21739733219146729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,1,0.007989333321650824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,1,0.009183999771873156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,3,0.007818666597207388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,16383,0.7036373615264893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,16383,0.4218133290608724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,3,0.009248000259200731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,7,0.007823999971151352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,7,0.009194666519761086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,15,0.008000000069538752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,15,0.009152000149091085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,31,0.008016000191370646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,31,0.009408000235756239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,63,0.008896000062425932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,63,0.009952000031868616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,127,0.00891733355820179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,127,0.011472000430027643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,255,0.010064000263810158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,255,0.011472000430027643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,511,0.010079999764760336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,511,0.011626667032639185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,1023,0.01584533353646596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,1023,0.015669333438078564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,2047,0.016986666868130367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,2047,0.01664000004529953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,4095,0.02754133443037669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,4095,0.027642667293548584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,8191,0.03826666623353958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,8191,0.037087999284267426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,16383,0.07464000085989635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,16383,0.05518400172392527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,32767,0.1243946651617686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,32767,0.09574400385220845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,65535,0.16775999466578165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,65535,0.2254506746927897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,1,0.014069333672523499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,7,0.014533333480358124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,1,0.014245333770910898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,3,0.01394133393963178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,15,0.015157333264748255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,3,0.01413333291808764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,7,0.014538666854302088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,31,0.018426666657129925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,15,0.015050667027632395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,131071,0.4355733394622803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,31,0.0179626668492953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,127,0.018778666853904724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,131071,0.32126400868097943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,63,0.018197332819302876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,63,0.018405333161354065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,127,0.018511999398469925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,1023,0.05884799857934316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,255,0.023013333479563396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,255,0.021967999637126923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,511,0.04275199770927429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,511,0.03363200028737386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,1023,0.09852799773216248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,2047,0.18473599354426065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,2047,0.11746133367220561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,4095,0.22080532709757486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,4095,0.35608001550038654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,1,0.024293333292007446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,1,0.023946667710940044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,8191,0.7217600345611572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,3,0.02382933348417282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,3,0.024469333390394848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,7,0.024847999215126038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,7,0.025306666890780132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,15,0.025450666745503742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,8191,0.43859732151031494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,15,0.026015999416510265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,31,0.03166399896144867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,31,0.03202133377393087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,63,0.03190399954716364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,63,0.032485333581765495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,127,0.033600000043710075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,127,0.0323786661028862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,255,0.0476693312327067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,255,0.040421334405740104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,511,0.07638933261235555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,511,0.06502399841944377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,1023,0.1851360003153483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,1023,0.1086079974969228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,2047,0.22427199284235635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,2047,0.36000001430511475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,1,0.043285335103670754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,1,0.044341335693995156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,3,0.04342400034268697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,3,0.04381333291530609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,7,0.04509866734345754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,7,0.04582933088143667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,15,0.046223998069763184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,15,0.04749333361784617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,31,0.05826666454474131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,31,0.05952533086140951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,63,0.06081066528956095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,63,0.059232001503308616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,127,0.06691200037797292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,127,0.061039999127388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,255,0.08435199658075969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,255,0.08017600079377492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,511,0.13818132877349854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,511,0.12283733487129211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,1,0.007871999715765318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,1023,0.3575146595637004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,1,0.009162666896979014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,3,0.007818666597207388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,3,0.00919999989370505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,7,0.007733333234985669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,1023,0.20458666483561197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,7,0.009285333255926767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,15,0.00795199970404307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,15,0.009205333267649015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,31,0.007983999947706858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,31,0.009269333134094873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,63,0.008661333471536636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,63,0.009786666681369146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,127,0.009093333035707474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,127,0.01166933278242747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,255,0.009088000282645226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,255,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,511,0.010015999898314476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,511,0.011488000551859537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,1023,0.016176000237464905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,1023,0.016048000504573185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,2047,0.027877333263556164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,2047,0.02733866622050603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,4095,0.03572800010442734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,4095,0.03469866762558619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,8191,0.06530666848023732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,8191,0.048207998275756836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,16383,0.11308800180753072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,16383,0.08377599716186523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,32767,0.14816000064214072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,32767,0.20150933663050333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,65535,0.3835093180338542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,1,0.08160000046094258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,65535,0.2734346588452657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,1,0.08336533109347026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,3,0.08101333181063335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,3,0.08281599978605907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,7,0.08458667000134786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,7,0.08649067083994548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,15,0.08827733000119527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,131071,0.43908798694610596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,15,0.08950933814048767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,131071,0.7124053637186686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,31,0.11428800225257874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,31,0.1123413344224294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,63,0.12109333276748657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,63,0.11821333567301433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,127,0.12455466389656067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,127,0.12320533394813538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,255,0.15819199879964194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,255,0.15060800313949585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,511,0.2605440020561218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,511,0.23331199089686075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,float16,1,0.15676800409952799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,fp8,1,0.1606666644414266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,float16,3,0.1570186714331309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,fp8,3,0.1607306698958079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,float16,7,0.1641973356405894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,fp8,7,0.1660533348719279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,float16,15,0.18360533316930136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,fp8,15,0.17352000872294107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,float16,31,0.2262986699740092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,fp8,31,0.22777066628138223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,float16,63,0.23379733165105185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,fp8,63,0.23543467124303183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,float16,127,0.23988266785939535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,1,0.008101333553592363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,fp8,127,0.2370026707649231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,1,0.009253333633144697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,3,0.007925333455204964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,3,0.009296000003814697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,7,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,7,0.008122666428486506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,float16,255,0.30186132589975995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,fp8,255,0.29335999488830566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,15,0.007930666829148928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,15,0.009269333134094873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,31,0.008234666660428047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,31,0.009354666496316591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,63,0.008976000050703684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,63,0.01002133327225844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,127,0.009109333157539368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,127,0.011621333658695221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,255,0.00921066664159298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,255,0.011567999919255575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,511,0.010399999717871347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,511,0.011839999506870905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,1023,0.026949333647886913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,1023,0.02608533451954524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,2047,0.031498665610949196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,2047,0.030426666140556335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,4095,0.05449066559473673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,4095,0.04106133431196213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,8191,0.10807466506958008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,8191,0.0668639987707138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,32767,0.3640906810760498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,16383,0.1926400065422058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,16383,0.125408003727595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,1,0.008570666735370954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,1,0.00877333308259646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,3,0.008736000085870424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,3,0.00901333304742972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,7,0.008799999952316284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,32767,0.2255573272705078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,7,0.00903466654320558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,65535,0.4373439947764079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,15,0.009402666861812273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,65535,0.715989351272583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,15,0.00966933307548364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,31,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,31,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,63,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,63,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,127,0.011482667177915573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,127,0.011509332805871964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,255,0.013210666676362356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,255,0.013093333691358566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,511,0.018522666146357853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,511,0.01926400015751521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,1023,0.05279466509819031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,1023,0.02842666705449422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,2047,0.0958666702111562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,2047,0.061381335059801735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,4095,0.1809813380241394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,4095,0.11310933033625285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,8191,0.21495999892552695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,8191,0.35417600472768146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,1,0.008767999708652496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,1,0.008869333192706108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,3,0.008725333337982496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,3,0.008965333302815756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,16383,0.4230133295059204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,16383,0.7083040078481039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,7,0.008656000097592672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,7,0.008922666932145754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,15,0.009002666920423508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,15,0.00914666677514712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,63,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,31,0.009413333609700203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,31,0.009882666791478792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,63,0.011418666690587997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,127,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,127,0.011322667201360067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,32767,1.2083626588185628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,32767,1.55459197362264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,255,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,255,0.011749333391586939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,511,0.01551466683546702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,511,0.015285332997639975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,1023,0.016496000190575916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,1023,0.016255999604860943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,2047,0.01657066618402799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,2047,0.01628799984852473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,4095,0.016538667182127636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,4095,0.016666666915019352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,8191,0.027621333797772724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,8191,0.02736533433198929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,16383,0.03800000001986822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,16383,0.0373279998699824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,32767,0.0784800002972285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,1,0.0086666668454806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,32767,0.057962665955225624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,65535,0.1409173309803009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,1,0.008842666943868002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,3,0.00874133345981439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,7,0.009018666421373686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,3,0.008885333314538002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,7,0.008602666358153025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,65535,0.11166399717330933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,31,0.00983466642598311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,15,0.008912000184257826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,15,0.00915733352303505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,31,0.009717333440979322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,63,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,127,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,63,0.011434666812419891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,127,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,255,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,255,0.011413333316644033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,131071,0.2616639931996663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,511,0.015413332730531693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,131071,0.20267732938130698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,511,0.01563199982047081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,1023,0.016597333053747814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,1023,0.016271999726692837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,2047,0.01661866654952367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,2047,0.016522667060295742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,4095,0.027589333554108936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,16383,0.056874667604764305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,4095,0.027237333357334137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,8191,0.03835200021664301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,8191,0.037258667250474296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,16383,0.07818666597207387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,32767,0.13686399658521017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,32767,0.10859733819961548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,1,0.013888000200192133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,1,0.014384000251690546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,65535,0.1942346692085266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,65535,0.24997333685557047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,7,0.014442666123310724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,15,0.015306666493415833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,3,0.014032000054915747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,3,0.014426667243242264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,7,0.014096000542243322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,131071,0.47729067007700604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,15,0.015610666324694952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,63,0.018746666610240936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,31,0.01876266673207283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,131071,0.36188801129659015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,31,0.01863466699918111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,63,0.018645333747069042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,127,0.019007999449968338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,127,0.019013332823912304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,255,0.02351466566324234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,255,0.02229333420594533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,511,0.04365866879622141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,511,0.03370666752258936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,2047,0.11628799637158711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,1023,0.09831999739011128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,1023,0.05964266757170359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,2047,0.18503999710083008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,4095,0.21822933355967203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,4095,0.3548746506373088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,1,0.008890666688481966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,1,0.009098666409651438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,8191,0.4184533357620239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,8191,0.697493314743042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,3,0.008954666554927826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,3,0.009098666409651438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,7,0.00873066671192646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,7,0.00890666681031386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,15,0.009056000038981438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,15,0.009183999771873156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,16383,0.8295253117879232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,31,0.009775999933481216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,31,0.009818666925032934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,127,0.011413333316644033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,16383,1.3962079683939617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,63,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,63,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,511,0.01543466622630755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,1023,0.01647466669480006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,127,0.011365332951148352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,255,0.011658667276302973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,255,0.011503999431928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,511,0.015370666980743408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,8191,0.07459733386834462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,1023,0.016672000288963318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,2047,0.02739733209212621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,2047,0.027215999861558277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,4095,0.038015998899936676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,4095,0.03702399879693985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,8191,0.05483733117580414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,16383,0.12562132875124613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,16383,0.09920533498128255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,32767,0.2267679969469706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,32767,0.1743733286857605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,1,0.013738666971524557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,1,0.014101333916187286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,3,0.014106666048367819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,65535,0.4325706561406453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,3,0.01440000037352244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,65535,0.3328319986661275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,7,0.014655999839305878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,7,0.015013333410024643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,15,0.017840000490347546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,15,0.0180479995906353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,31,0.017877332866191864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,31,0.01820266619324684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,63,0.018191999445358913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,63,0.01829333355029424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,127,0.02258133391539256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,127,0.021525333325068157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,255,0.042362665136655174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,255,0.0334346666932106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,131071,0.8507253328959147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,131071,0.6472586790720621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,511,0.09830400347709656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,511,0.05938133100668589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,1023,0.18499199549357095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,1023,0.11805867155392964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,2047,0.22366400559743246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,2047,0.3543146848678589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,1,0.023717333873112995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,1,0.02426133304834366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,3,0.02456533412138621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,4095,0.4310133457183838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,7,0.02536533276240031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,3,0.025199999411900837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,4095,0.6968639691670736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,7,0.025861332813898723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,15,0.031717332700888314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,15,0.03203733265399933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,31,0.032032000521818794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,31,0.03169599920511246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,8191,0.8650080362955729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,63,0.03385599950949351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,63,0.032042667269706726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,8191,1.4296587308247883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,127,0.04757866760094961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,127,0.04108799993991852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,255,0.07674666742483775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,255,0.06566399832566579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,511,0.11028800408045451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,511,0.185914675394694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,1023,0.3601226806640625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,1023,0.24084800481796265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,1,0.04265599946180979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,1,0.04444799820582072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,2047,0.6981973648071289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,2047,0.4356906811396281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,3,0.04470400015513102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,3,0.04586666822433472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,7,0.04643733302752177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,7,0.048384000857671104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,15,0.05864533285299937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,15,0.05884266893068949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,63,0.06264000137646993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,31,0.062362665931383766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,31,0.059392000238100685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,63,0.06659733255704244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,127,0.08534933129946391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,127,0.08121066788832347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,255,0.1400373379389445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,255,0.1234933336575826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,1,0.00874133345981439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,511,0.3582666714986165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,511,0.20917334159215292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,1,0.009162666896979014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,3,0.008986666798591614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,3,0.00895999992887179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,1023,0.7047893206278483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,15,0.009306666751702627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,1023,0.4352480173110962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,7,0.00895999992887179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,7,0.009189333145817121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,15,0.009162666896979014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,31,0.009589333087205887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,31,0.0099093330403169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,255,0.011711999773979187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,63,0.011338666081428528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,63,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,127,0.011488000551859537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,127,0.011589333415031433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,255,0.011440000186363855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,511,0.016437333077192307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,511,0.01609066625436147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,1023,0.027621333797772724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,4095,0.06482666730880737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,1023,0.027066667874654133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,2047,0.03676799933115641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,2047,0.034517332911491394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,4095,0.0488373339176178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,8191,0.08604266246159871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,8191,0.112527996301651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,16383,0.15467733144760132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,16383,0.2030186653137207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,32767,0.380570650100708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,32767,0.28702400128046673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,65535,0.7414506276448568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,65535,0.5477706591288248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,1,0.08103999992211659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,1,0.083146666487058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,3,0.0867786705493927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,3,0.08329600095748901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,7,0.08813866972923279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,7,0.09025599559148152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,15,0.11750400066375732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,15,0.11197866996129353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,31,0.12230400244394939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,31,0.11913067102432251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,63,0.12559466560681662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,63,0.12386666735013326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,127,0.15996266404787698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,131071,0.8929119904836019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,127,0.1530400017897288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,255,0.2355253299077352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,131071,1.3975733121236165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,255,0.26622400681177777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,511,0.4016053279240926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,511,0.7003520329793295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,1,0.1578986644744873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,fp8,1,0.16077333688735962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,3,0.16914133230845133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,fp8,3,0.1678826610247294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,7,0.1887306571006775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,fp8,7,0.1819253365198771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,15,0.23035200436909994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,fp8,15,0.23376532395680746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,31,0.23758933941523233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,fp8,31,0.2365600069363912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,63,0.2405866583188375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,1,0.00902399979531765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,1,0.009258666386206945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,fp8,63,0.23823465903600058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,3,0.009039999917149544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,3,0.009306666751702627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,7,0.009045333291093508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,127,0.307584007581075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,7,0.00922133338948091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,fp8,127,0.29729066292444867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,15,0.009285333255926767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,15,0.00943999985853831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,31,0.009994666402538618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,31,0.01020800011853377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,255,0.5098133484522501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,63,0.011546666423479715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,63,0.011626667032639185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,fp8,255,0.4645173152287801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,127,0.011722666521867117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,127,0.011637333780527115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,255,0.012181332955757776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,255,0.011701333026091257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,511,0.02651199946800868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,511,0.02593066543340683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,1023,0.0312266672650973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,1023,0.030373332401116688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,2047,0.05409066875775655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,2047,0.04197866717974345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,4095,0.107424000898997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,8191,0.13108266393343607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,4095,0.06936533252398173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,8191,0.19298666715621948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,16383,0.2374346653620402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,16383,0.36019734541575116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,1,0.008117333054542542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,1,0.009242666885256767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,3,0.009173333023985228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,3,0.008047999814152718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,7,0.007925333455204964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,32767,0.4509226481119792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,7,0.009152000149091085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,32767,0.7006666660308838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,15,0.007989333321650824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,15,0.00902399979531765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,31,0.008186666915814081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,31,0.009328000247478485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,63,0.008858666444818178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,127,0.011663999408483505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,63,0.010064000263810158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,127,0.00901333304742972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,255,0.009232000137368837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,255,0.011557333171367645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,511,0.010362666721145311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,1023,0.025936000049114227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,65535,1.4055733680725098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,65535,0.883466641108195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,511,0.011749333391586939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,1023,0.029098667204380035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,2047,0.03142400085926056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,2047,0.03057066599527995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,4095,0.05499733487764994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,4095,0.041690667470296226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,8191,0.06678933401902516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,8191,0.10796266794204712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,16383,0.1923253337542216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,16383,0.1260640025138855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,1,0.007696000238259633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,1,0.008810666700204214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,3,0.007711999739209811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,3,0.008992000172535578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,7,0.007567999884486198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,7,0.00891733355820179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,32767,0.22567999362945557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,32767,0.3626240094502767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,15,0.00766933336853981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,15,0.00890666681031386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,31,0.007941333577036858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,31,0.009050666665037474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,63,0.008613333106040955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,63,0.00979200005531311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,127,0.00874133345981439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,127,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,255,0.008810666700204214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,255,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,511,0.009717333440979322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,2047,0.016544000556071598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,511,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,1023,0.01545599972208341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,1023,0.015226667126019796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,2047,0.01642666632930438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,65535,0.7142453193664551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,4095,0.016575999557971954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,65535,0.43213868141174316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,4095,0.016650666793187458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,8191,0.016575999557971954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,8191,0.017573333034912746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,16383,0.016544000556071598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,16383,0.016544000556071598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,32767,0.017535999417304993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,32767,0.017898666361967724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,1,0.007530666887760162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,1,0.008890666688481966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,3,0.007696000238259633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,3,0.009002666920423508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,7,0.007520000139872233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,7,0.008863999818762144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,65535,0.023973333338896435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,15,0.007711999739209811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,15,0.009957333405812582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,63,0.009663999701539675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,65535,0.02199999988079071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,31,0.007776000226537387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,31,0.009039999917149544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,63,0.008559999987483025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,127,0.008805333326260248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,127,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,511,0.00972800018886725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,255,0.008645333349704742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,255,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,131071,0.05665066838264465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,511,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,131071,0.032042667269706726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,1023,0.01569066693385442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,1023,0.01565333331624667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,2047,0.016544000556071598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,2047,0.016421332955360413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,4095,0.016842667013406754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,4095,0.016271999726692837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,8191,0.016778666526079178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,8191,0.016773333152135212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,16383,0.027552001178264618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,16383,0.02736533433198929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,32767,0.03860266755024592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,32767,0.03716266651948293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,1,0.00871999996403853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,1,0.008762666955590248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,3,0.008639999975760778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,3,0.008709333216150602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,65535,0.07903466622034709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,7,0.009226666763424873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,7,0.008896000062425932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,15,0.008821333448092142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,65535,0.05709333221117655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,15,0.009541333342591921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,31,0.009477333476146063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,31,0.009466666728258133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,63,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,127,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,63,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,127,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,255,0.01138666644692421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,255,0.011391999820868174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,511,0.012938667088747025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,511,0.01333333303531011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,1023,0.018853332847356796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,1023,0.018181333939234417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,2047,0.028944000601768494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,2047,0.05277866621812185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,4095,0.09648533662160237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,131071,0.1415786643822988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,4095,0.06080533564090729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,131071,0.10898666580518086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,8191,0.11261866490046184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,8191,0.1827359994252523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,1,0.007829333345095316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,1,0.00902399979531765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,3,0.008000000069538752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,3,0.009066666786869368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,7,0.007850666840871176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,16383,0.3577706813812256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,7,0.009077333534757296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,15,0.007754666730761528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,16383,0.2136639952659607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,15,0.009098666409651438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,31,0.008101333553592363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,31,0.009173333023985228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,63,0.00860799973209699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,63,0.00983466642598311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,127,0.008890666688481966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,127,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,255,0.008885333314538002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,255,0.01138666644692421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,511,0.009919999788204828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,511,0.011600000162919363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,32767,0.5928320089975992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,1023,0.01588800052801768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,32767,0.7770400047302246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,1023,0.01579733317097028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,2047,0.016650666793187458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,2047,0.01643199970324834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,4095,0.016976000120242436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,4095,0.016879999389251072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,8191,0.02788266787926356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,8191,0.027087998886903126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,16383,0.03864533454179764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,32767,0.05685866872469584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,16383,0.03750933210055033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,32767,0.07798400024573009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,1,0.014240000396966934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,1,0.01414399966597557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,3,0.014159999787807465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,65535,0.13545599579811096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,3,0.013978666315476099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,7,0.014432000617186228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,7,0.014352000008026758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,15,0.014346666634082794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,65535,0.10579733053843181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,15,0.01451733335852623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,31,0.01526933287580808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,31,0.015754666179418564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,63,0.01854933301607768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,63,0.018810667097568512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,127,0.01887999971707662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,127,0.018826667219400406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,255,0.018895999838908512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,131071,0.18522665898005167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,255,0.018960000326236088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,511,0.023578666150569916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,131071,0.2476266622543335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,511,0.022255999346574146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,1023,0.044122666120529175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,1023,0.03345600018898646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,2047,0.0598880002895991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,2047,0.09839466214179993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,4095,0.18608532349268594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,4095,0.1165706713994344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,1,0.013855999956528345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,1,0.014218666901191076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,3,0.014127999544143677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,8191,0.2177493373552958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,8191,0.3566399812698364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,3,0.014010666559139887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,7,0.01431999976436297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,7,0.014778666198253632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,15,0.014730667074521383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,15,0.014981333166360855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,31,0.018085333208243053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,31,0.01828266680240631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,63,0.018298666924238205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,63,0.018432000031073887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,127,0.018373332917690277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,511,0.04298133154710134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,127,0.018730666488409042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,16383,0.41971735159556073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,16383,0.7024213473002116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,255,0.023082666099071503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,255,0.022096000611782074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,511,0.03377600014209747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,2047,0.11734933654467265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,1023,0.0978613297144572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,1023,0.05904000004132589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,2047,0.18449066082636514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,1,0.024512000381946564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,4095,0.3563573360443115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,1,0.023775999744733173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,4095,0.22046399116516113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,3,0.02382933348417282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,3,0.024319998919963837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,7,0.02459733436505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,7,0.025290665527184803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,15,0.025477332373460133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,15,0.02587199956178665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,31,0.03179733455181122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,31,0.032127998769283295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,63,0.03175999969244003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,63,0.03215466688076655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,127,0.03389333436886469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,127,0.03257599969704946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,255,0.04738666613896688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,255,0.041189332803090416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,511,0.0764160007238388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,1,0.007903999959429106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,511,0.06578133503595988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,1,0.009136000027259191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,3,0.007781333600481351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,3,0.009328000247478485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,7,0.009093333035707474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,1023,0.10897599657376607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,1023,0.1856693426767985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,7,0.007903999959429106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,2047,0.22401066621144614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,2047,0.3577173153559367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,15,0.007829333345095316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,127,0.008837333569924036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,15,0.009109333157539368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,31,0.008181333541870117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,31,0.009248000259200731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,511,0.011551999797423681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,63,0.009050666665037474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,63,0.009866666669646898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,127,0.011424000064531961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,255,0.00903466654320558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,255,0.011424000064531961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,4095,0.02771199991305669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,511,0.00996800015370051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,1023,0.01573866605758667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,1023,0.01575999955336253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,2047,0.017871999492247898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,2047,0.01672533278663953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,4095,0.027514666318893433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,8191,0.03853866706291834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,8191,0.037104000647862755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,16383,0.07374399900436401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,16383,0.0551093320051829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,32767,0.12366933623949687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,32767,0.09654933214187622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,1,0.04313066601753235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,1,0.04391466577847799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,3,0.04354666670163473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,65535,0.16778133312861124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,65535,0.22389866908391318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,3,0.044341335693995156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,7,0.04586133360862732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,7,0.04470933477083842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,15,0.04664533336957296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,31,0.05843733251094818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,15,0.047237331668535866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,131071,0.43480531374613446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,131071,0.3208799958229065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,31,0.05898666878541311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,63,0.05972266693909963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,63,0.06076799829800924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,127,0.06709333260854085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,127,0.06200533111890157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,255,0.08487466971079509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,255,0.08046400050322215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,511,0.13847466309865317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,511,0.12325867017110188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,1023,0.3572160005569458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,1023,0.20731200774510702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,1,0.08094933132330577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,1,0.08310933411121368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,3,0.0817386656999588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,3,0.08281066517035167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,7,0.08454933762550354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,15,0.08823466300964355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,15,0.0885706643263499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,31,0.11478933691978455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,31,0.11285332838694255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,63,0.1206773320833842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,63,0.11838400363922119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,127,0.12500799695650736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,127,0.12316800157229106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,1,0.00784533346692721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,255,0.15826666355133057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,255,0.15240533153216043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,1,0.009152000149091085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,3,0.007871999715765318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,3,0.009189333145817121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,7,0.007871999715765318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,15,0.00903466654320558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,511,0.2357599933942159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,7,0.00915733352303505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,15,0.008021333565314611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,511,0.2598186731338501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,31,0.008293333152929941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,31,0.009242666885256767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,63,0.008842666943868002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,127,0.00895999992887179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,63,0.010010666524370512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,511,0.010053333515922228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,127,0.011551999797423681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,255,0.011482667177915573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,255,0.009194666519761086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,511,0.011482667177915573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,1023,0.016330666840076447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,1023,0.01597333326935768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,2047,0.027717334528764088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,2047,0.027424000203609467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,4095,0.03667200108369192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,4095,0.03446399917205175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,8191,0.06426666676998138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,8191,0.04870399832725525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,16383,0.11272533734639485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,16383,0.08412266770998637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,1,0.009066666786869368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,32767,0.20265599091847739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,32767,0.14808000127474466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,1,0.009258666386206945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,3,0.009093333035707474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,3,0.009322666873534521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,7,0.009072000160813332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,7,0.009392000113924345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,15,0.00922133338948091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,15,0.009541333342591921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,31,0.010037333394090334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,65535,0.27374400695164997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,65535,0.38173866271972656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,31,0.010064000263810158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,63,0.011733333269755045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,63,0.011727999895811081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,255,0.01210133358836174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,127,0.011663999408483505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,127,0.011733333269755045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,255,0.011802667131026586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,511,0.02646933247645696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,511,0.025978667040665943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,1023,0.03107200066248576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,7,0.08590400218963623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,1023,0.03018666555484136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,2047,0.05482666691144308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,2047,0.04162666698296865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,4095,0.1071573297182719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,131071,0.4362613360087077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,16383,0.3617226680119832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,4095,0.07095466554164886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,131071,0.7139360109965006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,8191,0.19139200448989868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,8191,0.13132266203562418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,16383,0.23759466409683228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,1,0.008767999708652496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,1,0.009002666920423508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,3,0.00873066671192646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,3,0.008863999818762144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,7,0.008762666955590248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,7,0.008949333180983862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,15,0.008842666943868002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,15,0.009183999771873156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,31,0.009488000224033991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,31,0.009786666681369146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,32767,0.45101332664489746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,32767,0.700106700261434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,63,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,63,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,127,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,127,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,255,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,255,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,511,0.015141333142916361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,511,0.015103999525308609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,65535,0.8842826684316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,1023,0.01626666635274887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,65535,1.4055946667989094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,1023,0.01639466608564059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,4095,0.01646399994691213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,2047,0.016293333222468693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,2047,0.016208000481128693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,4095,0.016442666451136272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,8191,0.01661866654952367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,8191,0.016656000167131424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,16383,0.016810666769742966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,16383,0.01722666621208191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,32767,0.022805333137512207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,32767,0.02179199953873952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,1,0.008703999842206636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,1,0.00873066671192646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,3,0.008602666358153025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,3,0.00902399979531765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,65535,0.05593599875768026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,65535,0.03190933416287104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,7,0.008517333616813024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,7,0.008832000195980072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,15,0.00901333304742972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,15,0.009098666409651438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,31,0.00960533320903778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,31,0.009557333464423815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,63,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,63,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,127,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,131071,0.09983999530474345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,127,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,255,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,255,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,511,0.01543466622630755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,131071,0.06758933266003926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,511,0.015279999623696009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,1023,0.016469333320856094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,1023,0.016314666718244553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,2047,0.01640533283352852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,2047,0.016399999459584553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,4095,0.01684800038735072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,4095,0.016554666062196095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,8191,0.027456000447273254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,8191,0.02701333413521449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,16383,0.03817066550254822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,16383,0.03731200098991394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,32767,0.0792906681696574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,32767,0.05676266551017761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,1,0.008629333227872849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,1,0.00873066671192646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,3,0.008714666590094566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,3,0.008863999818762144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,7,0.008767999708652496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,7,0.008767999708652496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,15,0.009541333342591921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,15,0.009429333110650381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,65535,0.14173866311709085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,65535,0.11195199688275655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,31,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,127,0.011626667032639185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,31,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,63,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,63,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,127,0.011391999820868174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,255,0.013450667262077332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,255,0.013013333082199097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,511,0.018325333793958027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,511,0.01814933369557063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,131071,0.2619306643803914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,2047,0.061706667145093284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,1023,0.052426666021347046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,1023,0.028538666665554047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,131071,0.20065067211786905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,2047,0.09589866797129314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,4095,0.18179200092951456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,8191,0.2148853341738383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,8191,0.3548479874928792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,4095,0.11359467109044392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,1,0.008821333448092142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,1,0.009008000294367472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,3,0.008639999975760778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,3,0.008842666943868002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,7,0.008639999975760778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,16383,0.7087466716766357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,16383,0.4208693504333496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,7,0.00878399983048439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,15,0.008842666943868002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,15,0.009056000038981438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,31,0.009557333464423815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,31,0.009722666814923286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,127,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,63,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,63,0.011407999942700068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,127,0.011322667201360067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,255,0.011477333803971609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,255,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,511,0.015450666348139444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,511,0.015413332730531693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,1023,0.0164533331990242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,1023,0.016415999581416447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,32767,1.2091413338979085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,32767,1.5515413284301758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,2047,0.016757333030303318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,2047,0.016965333372354507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,8191,0.03743999948104223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,4095,0.02788266787926356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,4095,0.02762666592995326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,8191,0.03857066730658213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,16383,0.07831466694672902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,16383,0.05677333474159241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,32767,0.13742400209108988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,32767,0.10989866654078166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,1,0.013760000467300415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,1,0.014303999642531076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,3,0.014175999909639359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,3,0.014352000008026758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,65535,0.25042666991551715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,7,0.014175999909639359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,65535,0.19370132684707642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,7,0.014362666755914688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,15,0.01522133375207583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,15,0.015541333705186844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,31,0.018709332992633183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,63,0.018725333114465077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,31,0.01870399961868922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,63,0.01854933301607768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,127,0.01913600042462349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,127,0.019082666685183842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,131071,0.36076800028483075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,131071,0.47787733872731525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,255,0.023786666492621105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,255,0.022410665949185688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,511,0.04355733096599579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,511,0.03405333310365677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,1023,0.09855467081069946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,1023,0.05975999931494395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,2047,0.1163040002187093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,2047,0.18601065874099731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,4095,0.35603201389312744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,4095,0.218341330687205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,1,0.014090667168299357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,1,0.013807999591032663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,3,0.014085333794355392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,3,0.014389333625634512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,8191,0.41970666249593097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,8191,0.6992959976196289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,7,0.014805333067973455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,7,0.014896000425020853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,15,0.017946666727463405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,15,0.01803733284274737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,31,0.018042666216691334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,63,0.018160000443458557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,31,0.018266666680574417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,63,0.01821333294113477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,127,0.023024000227451324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,127,0.02143999934196472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,255,0.04240000247955322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,255,0.033930666744709015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,511,0.09784000118573506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,511,0.05938133100668589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,16383,1.3976000150044758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,16383,0.8277440071105957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,1023,0.18426666657129923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,1023,0.11744532982508342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,2047,0.35655999183654785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,2047,0.22339733441670737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,1,0.023605334262053173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,1,0.024293333292007446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,3,0.02459733436505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,3,0.025077333052953083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,7,0.02536533276240031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,4095,0.4296426773071289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,15,0.03193599979082743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,4095,0.6997333367665609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,7,0.0260959987839063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,15,0.03176533430814743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,31,0.032127998769283295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,31,0.031744000812371574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,63,0.03404266635576884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,63,0.03209600100914637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,127,0.047914668917655945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,127,0.04087999959786733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,255,0.07730666796366374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,255,0.0662720004717509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,511,0.18594666322072348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,511,0.10993066430091858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,1023,0.3584959904352824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,1023,0.22559465964635214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,1,0.00892800030608972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,1,0.008976000050703684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,7,0.008752000207702318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,3,0.008752000207702318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,15,0.009039999917149544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,2047,0.7030346393585205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,31,0.009839999799927076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,3,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,2047,0.43430399894714355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,7,0.009029333169261614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,15,0.009178666397929192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,31,0.009546666716535887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,63,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,63,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,127,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,127,0.011445333560307821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,255,0.01157333329319954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,255,0.011482667177915573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,511,0.015413332730531693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,511,0.015706667055686314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,1023,0.016496000190575916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,1023,0.016666666915019352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,2047,0.02759466568628947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,2047,0.027461332579453785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,4095,0.03810133288304011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,4095,0.03654933224121729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,8191,0.0747626672188441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,8191,0.055061335364977516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,16383,0.12578133742014566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,16383,0.099781334400177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,32767,0.2274613380432129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,32767,0.17391467094421387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,1,0.04288533329963684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,1,0.04450133442878723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,65535,0.4328693151473999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,3,0.044351999958356224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,65535,0.3340373436609904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,3,0.04602666695912679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,7,0.04654400050640106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,7,0.04744533201058706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,15,0.058277333776156105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,15,0.05913599828879038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,31,0.06281066437562306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,31,0.05942933261394501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,131071,0.8477439880371094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,127,0.08602133393287659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,63,0.06214400132497152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,63,0.06702933212121327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,131071,0.6455466747283936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,127,0.081386665503184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,255,0.14084266622861227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,255,0.12380799651145935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,511,0.3580319881439209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,511,0.2085919976234436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,1023,0.4374133348464966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,1023,0.7048959732055664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,1,0.08105066418647766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,1,0.0836906631787618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,3,0.0839519997437795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,3,0.08693333466847737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,7,0.08746666709582011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,7,0.08966400225957234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,15,0.11769599715868632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,15,0.11212799946467082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,31,0.12250133355458577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,31,0.12004266182581584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,63,0.1252906620502472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,63,0.12451199690500896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,1,0.008698666468262672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,127,0.16028799613316855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,127,0.15335999925931296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,1,0.009088000282645226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,3,0.008762666955590248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,3,0.008943999807039896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,7,0.008762666955590248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,7,0.009103999783595404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,15,0.009018666421373686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,15,0.009098666409651438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,31,0.009578666960199675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,255,0.2645813425381978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,31,0.009872000043590864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,63,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,255,0.2358506719271342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,63,0.011429333438475927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,127,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,127,0.011343999455372492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,255,0.011584000041087469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,255,0.011519999553759893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,511,0.403109351793925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,511,0.7007946968078613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,511,0.016255999604860943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,511,0.015824000040690105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,1023,0.02739199995994568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,1023,0.0273333340883255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,2047,0.035802667339642845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,2047,0.03446399917205175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,4095,0.048154667019844055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,4095,0.06362666686375935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,8191,0.11156266927719116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,8191,0.0853760043780009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,16383,0.20433600743611655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,16383,0.15425599614779154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,1,0.007786666974425316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,32767,0.28458666801452637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,32767,0.38213332494099933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,1,0.00901333304742972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,3,0.00784533346692721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,7,0.007850666840871176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,3,0.009253333633144697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,7,0.00901333304742972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,15,0.007818666597207388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,15,0.009162666896979014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,31,0.00808533343176047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,31,0.009226666763424873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,65535,0.546234647432963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,63,0.008826666822036108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,63,0.00978133330742518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,65535,0.743120034535726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,127,0.008954666554927826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,255,0.009077333534757296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,127,0.01138666644692421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,255,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,511,0.01002133327225844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,511,0.011600000162919363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,1023,0.0163680004576842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,1023,0.015856000284353893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,2047,0.02757866680622101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,2047,0.02755733331044515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,4095,0.035973332822322845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,131071,0.8927679856618246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,4095,0.03461333364248276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,8191,0.06457066535949707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,8191,0.04786666731039683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,131071,1.3972373008728027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,16383,0.11314666271209717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,16383,0.08321066697438557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,32767,0.20360533396402994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,32767,0.1499573290348053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,1,0.007733333234985669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,1,0.00895999992887179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,3,0.007653333246707916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,7,0.007823999971151352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,3,0.008912000184257826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,7,0.008954666554927826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,15,0.007754666730761528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,15,0.008901333436369896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,31,0.007920000081261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,31,0.009183999771873156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,63,0.008592000231146812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,63,0.009754666437705358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,127,0.00873066671192646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,127,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,65535,0.38230931758880615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,255,0.008863999818762144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,255,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,511,0.009712000067035357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,511,0.011429333438475927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,1023,0.015392000476519266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,2047,0.016549333930015564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,1023,0.015184000134468079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,65535,0.27293866872787476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,2047,0.01621333385507266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,4095,0.016757333030303318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,4095,0.01653333380818367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,8191,0.016501333564519882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,8191,0.016677333662907284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,16383,0.016544000556071598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,16383,0.016538667182127636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,32767,0.016832000265518825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,32767,0.016656000167131424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,1,0.007727999861041705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,1,0.00891733355820179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,3,0.007786666974425316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,3,0.00898133342464765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,65535,0.016901332885026932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,7,0.007791999727487564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,65535,0.017466666797796886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,7,0.008943999807039896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,131071,0.43672533830006915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,15,0.007680000116427739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,15,0.00898133342464765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,131071,0.716538667678833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,63,0.008458666503429413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,31,0.008058666562040647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,31,0.009130666653315226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,127,0.011365332951148352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,63,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,131071,0.022367998957633972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,127,0.008656000097592672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,255,0.00892800030608972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,131071,0.02201066662867864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,255,0.011349332829316458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,511,0.009813333551088968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,511,0.011407999942700068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,1023,0.015669333438078564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,1023,0.015216000378131866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,2047,0.01648533344268799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,2047,0.01646399994691213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,4095,0.016645333419243496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,4095,0.016336000214020412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,8191,0.016832000265518825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,8191,0.016517333686351776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,16383,0.01664000004529953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,16383,0.016634666671355564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,32767,0.017573333034912746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,32767,0.01695466662446658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,1,0.007877333089709282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,1,0.009322666873534521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,3,0.008042666440208754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,3,0.009088000282645226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,7,0.008117333054542542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,65535,0.02266666789849599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,7,0.009183999771873156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,15,0.007893333211541176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,65535,0.02205866575241089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,15,0.009317333499590555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,31,0.008303999900817871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,31,0.009392000113924345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,63,0.01007466639081637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,127,0.009162666896979014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,63,0.008762666955590248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,127,0.011642667154471079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,255,0.009343999748428663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,255,0.011727999895811081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,511,0.010373333469033241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,131071,0.05749333401521047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,511,0.011658667276302973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,131071,0.0315786674618721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,1023,0.026863999664783478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,1023,0.025983999172846477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,2047,0.03149333347876867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,2047,0.03031466652949651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,4095,0.0536053329706192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,4095,0.04148799926042557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,8191,0.10666666428248088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,8191,0.06781866649786632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,1,0.007871999715765318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,16383,0.1252959966659546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,1,0.009130666653315226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,16383,0.19205333789189658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,3,0.007797333101431529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,3,0.00914666677514712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,7,0.007807999849319458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,7,0.00902399979531765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,15,0.007786666974425316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,15,0.009130666653315226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,31,0.007967999825874964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,31,0.00933333362142245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,63,0.008725333337982496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,63,0.009850666547815004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,32767,0.2292799949645996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,127,0.008986666798591614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,127,0.011514666179815928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,32767,0.36291201909383136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,511,0.009946666657924652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,255,0.00903466654320558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,255,0.011482667177915573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,1023,0.015450666348139444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,511,0.01157333329319954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,1023,0.01578666642308235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,2047,0.01684800038735072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,2047,0.016757333030303318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,4095,0.016549333930015564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,4095,0.016672000288963318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,8191,0.016885332763195038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,8191,0.016496000190575916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,16383,0.027888000011444092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,16383,0.02716800073782603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,32767,0.03829866647720337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,65535,0.43249066670735675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,65535,0.7144320011138916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,32767,0.037477334340413414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,1,0.008565333361426989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,1,0.008634666601816813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,3,0.008703999842206636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,65535,0.05793599784374237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,65535,0.07834666470686595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,3,0.008650666723648706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,7,0.008778666456540426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,7,0.008810666700204214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,15,0.008858666444818178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,15,0.008938666433095932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,31,0.009541333342591921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,31,0.009477333476146063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,131071,0.10970133543014526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,63,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,63,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,127,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,131071,0.14235732952753702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,127,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,255,0.011503999431928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,255,0.011381333072980246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,511,0.013194666554530462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,511,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,1023,0.01851733277241389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,1023,0.018229333062966663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,2047,0.05236800014972687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,2047,0.02938666691382726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,4095,0.09568533301353455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,4095,0.061573331554730736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,8191,0.11216533184051514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,1,0.014170666535695394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,8191,0.18153599898020426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,1,0.014080000420411428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,3,0.014101333916187286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,16383,0.3571840127309163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,3,0.014208000153303146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,7,0.014256000518798828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,16383,0.21421867609024048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,7,0.014495999862750372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,15,0.014256000518798828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,15,0.014368000129858652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,31,0.015530666957298914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,31,0.015674666812022526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,127,0.01871466636657715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,63,0.01868266612291336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,127,0.018986667195955913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,63,0.018570666511853535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,255,0.01929066702723503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,255,0.01886933296918869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,511,0.02333866556485494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,511,0.022245332598686218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,1023,0.04423466821511587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,1023,0.0341386670867602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,2047,0.09787733356157939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,2047,0.059706668059031166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,32767,0.7758986949920654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,32767,0.604640007019043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,4095,0.18427199125289917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,4095,0.1167146662871043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,1,0.01393066719174385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,1,0.014101333916187286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,3,0.014032000054915747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,8191,0.3552159865697225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,7,0.014170666535695394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,3,0.014186666657527288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,7,0.014522666732470194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,15,0.014864000181357065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,15,0.015194666882356008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,8191,0.21689067284266153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,31,0.01791999985774358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,31,0.018405333161354065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,63,0.018144000321626663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,63,0.018437333405017853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,127,0.01858666663368543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,127,0.018842666099468868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,255,0.02310933421055476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,255,0.021989333132902782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,511,0.04278933505217234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,511,0.033701332906881966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,1023,0.09796266754468282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,1023,0.05968533456325531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,1,0.007807999849319458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,2047,0.116949329773585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,3,0.007834666719039282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,1,0.008986666798591614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,2047,0.1840160091718038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,3,0.009072000160813332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,7,0.007781333600481351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,7,0.008901333436369896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,15,0.007637333124876022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,15,0.008986666798591614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,4095,0.22032533089319864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,4095,0.3549013137817383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,31,0.007967999825874964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,31,0.009322666873534521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,63,0.008559999987483025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,63,0.009759999811649323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,511,0.011472000430027643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,127,0.008661333471536636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,127,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,255,0.00891733355820179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,255,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,511,0.009797333429257074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,1023,0.015909332782030106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,1023,0.01575999955336253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,2047,0.01669866715868314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,2047,0.016399999459584553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,4095,0.016672000288963318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,4095,0.016634666671355564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,8191,0.02779199928045273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,8191,0.027424000203609467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,16383,0.038405333956082664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,16383,0.03732266773780187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,32767,0.0784746656815211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,32767,0.05691199998060862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,65535,0.13570132851600647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,1,0.02404800057411194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,65535,0.10672000050544739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,1,0.024421334266662598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,3,0.024101334313551586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,3,0.024351999163627625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,7,0.02470933397610982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,7,0.025285333395004272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,15,0.02566933383544286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,15,0.02605866640806198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,31,0.03164266546567281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,131071,0.24637866020202637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,31,0.032138665517171226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,131071,0.18657066424687704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,63,0.03183466692765554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,63,0.03252266595760981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,127,0.033530667424201965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,127,0.03278400003910065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,255,0.04810666541258494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,255,0.04102933406829834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,511,0.06654400130112965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,511,0.0766186664501826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,1023,0.18663465976715088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,1023,0.10987200339635213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,float16,1,0.04339733223120371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,2047,0.35632534821828205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,fp8,1,0.044213334719340004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,2047,0.22476800282796225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,float16,3,0.04367466767628988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,fp8,3,0.04427733520666758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,float16,7,0.044997334480285645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,fp8,7,0.04613333443800608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,float16,15,0.04669333497683207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,fp8,15,0.04757333298524221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,float16,31,0.05796800057093302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,fp8,31,0.059418668349583946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,float16,63,0.06131199995676676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,fp8,63,0.05934933324654897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,float16,127,0.06720000008742015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,fp8,127,0.062181333700815834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,1,0.007903999959429106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,fp8,255,0.08061866462230682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,1,0.009237333511312803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,float16,511,0.13845866918563843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,3,0.007930666829148928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,3,0.00922133338948091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,fp8,511,0.12383466958999634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,7,0.007903999959429106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,7,0.009109333157539368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,15,0.007797333101431529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,15,0.009125333279371262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,float16,1023,0.3590879837671916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,31,0.008080000057816505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,31,0.009285333255926767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,63,0.008837333569924036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,63,0.009824000298976898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,fp8,1023,0.20882666110992432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,127,0.00895999992887179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,127,0.011493333925803503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,255,0.00902399979531765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,255,0.011434666812419891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,511,0.010010666524370512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,511,0.011445333560307821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,1023,0.015765332927306492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,1023,0.015578666081031164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,2047,0.016917333006858826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,2047,0.016549333930015564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,float16,255,0.08532266815503438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,4095,0.027621333797772724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,4095,0.027434666951497395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,8191,0.038218667109807335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,8191,0.03718933214743932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,16383,0.07471466561158498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,16383,0.0553653339544932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,32767,0.12311999996503194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,1,0.008810666700204214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,1,0.009088000282645226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,32767,0.09676800171534221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,3,0.00884799969693025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,3,0.009029333169261614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,7,0.00898133342464765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,7,0.008954666554927826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,15,0.009039999917149544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,65535,0.22470933198928833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,65535,0.169322669506073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,63,0.01145600030819575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,15,0.009216000015536943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,63,0.011322667201360067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,127,0.011445333560307821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,31,0.009797333429257074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,131071,0.3223946690559387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,31,0.009786666681369146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,127,0.011530666301647821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,255,0.012533333152532578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,255,0.01138666644692421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,131071,0.43355198701222736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,511,0.016197333733240765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,511,0.01595199977358182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,1023,0.027445333699385326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,1023,0.027322667340437572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,2047,0.03670933345953623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,2047,0.03414933383464813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,4095,0.06436266501744588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,4095,0.049072002371152244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,8191,0.11282666524251302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,16383,0.15530666708946228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,8191,0.08556800087292989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,16383,0.20403200387954712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,32767,0.38301865259806317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,32767,0.28516799211502075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,1,0.008586666857202848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,1,0.008869333192706108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,3,0.008672000219424566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,3,0.008752000207702318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,7,0.008656000097592672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,7,0.008752000207702318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,15,0.00891733355820179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,15,0.008949333180983862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,31,0.00955200009047985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,31,0.00960533320903778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,63,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,63,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,127,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,127,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,255,0.011365332951148352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,255,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,511,0.015066667149464289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,511,0.015290666371583939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,1023,0.016021333634853363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,1023,0.01598400001724561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,2047,0.016469333320856094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,65535,0.5480746825536092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,65535,0.7429599761962891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,2047,0.016250666230916977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,4095,0.01632533346613248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,4095,0.01643199970324834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,8191,0.016271999726692837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,8191,0.016677333662907284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,16383,0.016447999825080235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,16383,0.01664000004529953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,32767,0.01722666621208191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,32767,0.016544000556071598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,1,0.008645333349704742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,1,0.00871999996403853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,3,0.0085333331177632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,65535,0.02369600037733714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,65535,0.021962667504946392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,3,0.008816000074148178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,7,0.008805333326260248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,7,0.008752000207702318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,15,0.00879466657837232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,15,0.009109333157539368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,31,0.010117333382368088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,31,0.009599999835093817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,131071,0.8904266357421875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,63,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,131071,1.4016159375508626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,63,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,131071,0.05713599920272827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,131071,0.031850665807724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,127,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,127,0.01146666705608368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,255,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,255,0.01128000020980835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,511,0.015290666371583939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,511,0.01534933348496755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,1023,0.016415999581416447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,1023,0.016208000481128693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,2047,0.016501333564519882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,2047,0.0162773331006368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,4095,0.016447999825080235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,4095,0.01651200031240781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,8191,0.01661866654952367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,8191,0.01658133293191592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,16383,0.017184000462293625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,16383,0.016650666793187458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,32767,0.023765332996845245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,32767,0.02176533391078313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,1,0.009008000294367472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,1,0.009285333255926767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,65535,0.05671466886997223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,3,0.009205333267649015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,3,0.009242666885256767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,65535,0.032144000132878624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,15,0.009648000200589498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,7,0.009056000038981438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,7,0.009205333267649015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,15,0.009077333534757296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,31,0.009930666536092758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,31,0.010117333382368088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,63,0.01163200040658315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,63,0.01166933278242747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,127,0.011690666278203329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,127,0.011551999797423681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,131071,0.1002293328444163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,1023,0.031034665803114574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,255,0.012085333466529846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,255,0.011909333368142446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,131071,0.06838933130105336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,511,0.02658133457104365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,511,0.026074667771657307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,1023,0.03010133405526479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,2047,0.04161600023508072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,2047,0.054511999090512596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,4095,0.10611200332641602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,4095,0.06925333539644878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,8191,0.19177067279815674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,8191,0.13077867031097412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,16383,0.3604053258895874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,16383,0.23888534307479858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,1,0.008693333094318708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,3,0.008512000242869059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,1,0.008687999720374743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,32767,0.7012906869252523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,3,0.00895999992887179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,7,0.008863999818762144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,15,0.008997333546479544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,7,0.008687999720374743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,32767,0.45187199115753174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,15,0.009109333157539368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,31,0.009413333609700203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,31,0.009690666571259499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,63,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,255,0.011333333949247995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,63,0.011391999820868174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,127,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,511,0.01552533358335495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,65535,1.4022026062011719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,127,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,255,0.011343999455372492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,1023,0.016384000579516094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,65535,0.879210631052653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,2047,0.01635733370979627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,511,0.015247999380032221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,1023,0.01621866722901662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,4095,0.016586666305859882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,2047,0.016341333587964375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,4095,0.01669866715868314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,16383,0.03821333249409994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,8191,0.027589333554108936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,8191,0.027301333844661713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,16383,0.03711999952793121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,32767,0.08004266520341237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,32767,0.05715199808279673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,1,0.008570666735370954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,1,0.008837333569924036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,3,0.008682666967312494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,3,0.008943999807039896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,65535,0.14131200313568115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,65535,0.1123306651910146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,7,0.00874133345981439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,7,0.008949333180983862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,15,0.009413333609700203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,15,0.009642666826645533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,31,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,131071,0.2620053291320801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,131071,0.20126932859420776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,31,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,127,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,63,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,255,0.013429333766301474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,63,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,127,0.011498666057984034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,255,0.013157332936922709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,511,0.018170667191346485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,511,0.018581333259741466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,1023,0.052602668603261314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,1023,0.028421332438786823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,2047,0.09609066446622212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,2047,0.0618453323841095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,4095,0.18419732650121054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,4095,0.11293333768844604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,8191,0.3543200095494588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,8191,0.21405333280563354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,1,0.013909333695967993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,1,0.014282666146755219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,3,0.014053333550691605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,3,0.014416000495354334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,7,0.014175999909639359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,7,0.0145066666106383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,16383,0.7079520225524902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,16383,0.41971198717753094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,15,0.015274666249752045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,15,0.01562133307258288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,31,0.018570666511853535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,31,0.01878400022784869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,63,0.018730666488409042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,63,0.018709332992633183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,127,0.019130667050679524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,127,0.018874666343132656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,255,0.023391999304294586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,255,0.022431999444961548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,511,0.04369066655635834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,511,0.03389866650104523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,1023,0.09902399778366089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,1023,0.060122668743133545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,2047,0.18458133935928345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,2047,0.11659733454386394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,32767,1.5528160730997722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,32767,1.1962186495463054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,1,0.0136266661187013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,4095,0.21730667352676392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,4095,0.35760533809661865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,1,0.014175999909639359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,3,0.014458666245142618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,7,0.014896000425020853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,3,0.014138666292031607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,7,0.014970666418472925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,15,0.018101333330074947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,15,0.018218666315078735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,8191,0.4198880195617676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,31,0.017909333109855652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,8191,0.7004960378011068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,63,0.01820266619324684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,31,0.018021332720915478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,63,0.01855466639002164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,127,0.023061332603295643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,127,0.02139200021823247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,255,0.04271999994913737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,255,0.03341866781314214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,511,0.059450666109720864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,511,0.0983733336130778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,1023,0.18454400698343912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,1,0.008698666468262672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,1023,0.11824533343315125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,1,0.00884799969693025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,3,0.008821333448092142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,3,0.009066666786869368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,7,0.00877333308259646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,7,0.00903466654320558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,2047,0.22243199745814005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,2047,0.35633599758148193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,15,0.008976000050703684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,15,0.00901333304742972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,31,0.009695999945203463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,31,0.009690666571259499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,63,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,63,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,127,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,127,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,511,0.015381333728631338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,4095,0.4301973183949788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,4095,0.7027040322621664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,255,0.011610666910807291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,255,0.011328000575304031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,511,0.015392000476519266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,1023,0.016352000335852306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,1023,0.016330666840076447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,2047,0.016656000167131424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,2047,0.016575999557971954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,4095,0.027482666075229645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,4095,0.027445333699385326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,8191,0.038405333956082664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,8191,0.03733866661787033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,16383,0.07878399888674419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,16383,0.05737066765626272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,32767,0.13818132877349854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,32767,0.10937066872914632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,65535,0.19316265980402628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,65535,0.25118400653203327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,1,0.023685333629449207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,1,0.024517332514127094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,3,0.02459733436505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,3,0.02510400116443634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,7,0.02536533276240031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,7,0.025909334421157837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,15,0.031850665807724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,15,0.03177600105603536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,31,0.031583999594052635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,131071,0.47564268112182617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,31,0.032069332897663116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,63,0.03408533334732056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,63,0.03241066634654999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,131071,0.36025067170461017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,127,0.048101335763931274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,127,0.040448000033696495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,255,0.06649599969387054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,255,0.07639466722806294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,511,0.1867093245188395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,511,0.11122133334477742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,1023,0.35683198769887287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,1023,0.2262986699740092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,float16,1,0.04306666553020477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,fp8,1,0.044362664222717285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,float16,3,0.044581333796183266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,2047,0.7007359663645426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,2047,0.4314773480097453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,fp8,3,0.04619733492533366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,float16,7,0.04659733176231384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,fp8,7,0.04966933528582255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,float16,15,0.058687999844551086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,fp8,15,0.05923733115196228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,float16,31,0.06260799864927928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,fp8,31,0.058874666690826416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,float16,63,0.0673333356777827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,fp8,63,0.0622026671965917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,float16,127,0.08668800195058186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,fp8,127,0.08124266564846039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,float16,255,0.13986666997273764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,1,0.00892800030608972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,1,0.008992000172535578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,fp8,255,0.1237600048383077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,3,0.008752000207702318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,3,0.00903466654320558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,7,0.008746666833758354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,15,0.00891733355820179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,7,0.009098666409651438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,float16,511,0.3596693277359009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,15,0.009253333633144697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,fp8,1023,0.43620800971984863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,31,0.009514666472872099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,63,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,fp8,511,0.20853867133458456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,31,0.009786666681369146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,float16,1023,0.7020746866861979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,63,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,127,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,127,0.011407999942700068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,255,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,255,0.01146666705608368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,511,0.015647999942302704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,511,0.015552000453074774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,1023,0.016682667036851246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,1023,0.016842667013406754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,2047,0.027488000690937042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,2047,0.027210667729377747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,4095,0.03862933317820231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,4095,0.03696533292531967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,8191,0.075573335091273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,8191,0.055439998706181846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,16383,0.1262506643931071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,16383,0.10001599788665771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,32767,0.22608532508214316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,32767,0.17566933234532675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,1,0.008005333443482717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,1,0.009125333279371262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,3,0.009194666519761086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,3,0.008010666817426682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,7,0.007818666597207388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,7,0.009119999905427298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,15,0.007920000081261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,15,0.009125333279371262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,31,0.00808533343176047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,65535,0.43302400906880695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,31,0.009279999881982803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,63,0.008767999708652496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,63,0.010005333150426546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,65535,0.33460799853007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,127,0.008965333302815756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,127,0.011445333560307821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,255,0.008949333180983862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,255,0.011519999553759893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,511,0.009930666536092758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,511,0.011482667177915573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,2047,0.017103999853134155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,1023,0.015893333901961643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,1023,0.015658666690190632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,2047,0.016602666427691776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,4095,0.02757333219051361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,4095,0.02735999971628189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,8191,0.03825066735347112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,8191,0.03696533292531967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,16383,0.07363200187683105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,16383,0.05594133337338766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,131071,0.849232037862142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,131071,0.6441386540730795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,32767,0.12483732899030049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,32767,0.09706667065620422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,1,0.008293333152929941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,1,0.008938666433095932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,3,0.007781333600481351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,3,0.008896000062425932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,7,0.007690666864315669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,7,0.008986666798591614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,15,0.007642666498819987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,15,0.008863999818762144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,31,0.007962666451931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,31,0.009061333412925402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,63,0.008581333483258883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,63,0.009797333429257074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,127,0.00860799973209699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,127,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,255,0.008752000207702318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,255,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,511,0.009749333063761393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,511,0.011322667201360067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,1023,0.015477333217859268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,65535,0.22392000754674277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,1023,0.015461333096027374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,2047,0.01637866720557213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,2047,0.016224000602960587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,65535,0.1706506609916687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,4095,0.01632000009218852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,4095,0.01621333385507266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,8191,0.016255999604860943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,8191,0.016506666938463848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,16383,0.01637866720557213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,16383,0.01635733370979627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,32767,0.016794666647911072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,1,0.007760000104705493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,32767,0.016842667013406754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,1,0.00892800030608972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,3,0.007680000116427739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,65535,0.016496000190575916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,3,0.009002666920423508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,7,0.007631999750932057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,7,0.008912000184257826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,65535,0.016506666938463848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,15,0.007781333600481351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,15,0.008933333059151968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,31,0.007920000081261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,31,0.00926399976015091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,63,0.008645333349704742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,63,0.009733333562811216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,127,0.008965333302815756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,127,0.011343999455372492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,255,0.008778666456540426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,511,0.009818666925032934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,255,0.011493333925803503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,131071,0.017055999487638474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,131071,0.43625601132710773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,511,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,1023,0.015301333119471868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,131071,0.017317333569129307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,131071,0.32731733719507855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,1023,0.015322666615247726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,4095,0.01637866720557213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,2047,0.016271999726692837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,2047,0.016176000237464905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,4095,0.016517333686351776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,8191,0.016501333564519882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,8191,0.016672000288963318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,16383,0.016645333419243496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,16383,0.016549333930015564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,32767,0.01657066618402799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,32767,0.016538667182127636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,1,0.007781333600481351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,1,0.009029333169261614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,3,0.007781333600481351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,3,0.009093333035707474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,7,0.007760000104705493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,7,0.008949333180983862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,65535,0.017386666188637417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,15,0.007797333101431529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,15,0.009109333157539368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,65535,0.01685333376129468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,31,0.008000000069538752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,31,0.009296000003814697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,63,0.008725333337982496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,63,0.00984533317387104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,127,0.00898133342464765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,127,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,255,0.00916800027092298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,131071,0.022543999056021374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,255,0.011482667177915573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,511,0.010015999898314476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,511,0.011605333536863327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,1023,0.016330666840076447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,131071,0.02187199890613556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,1023,0.015829333414634068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,2047,0.027834666272004444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,2047,0.027295999228954315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,4095,0.035946667194366455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,4095,0.03472533325354258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,8191,0.06376533210277557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,8191,0.04860266546408335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,16383,0.11270399888356526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,16383,0.08434666196505229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,1,0.007749333356817563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,1,0.008992000172535578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,3,0.007781333600481351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,3,0.00914666677514712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,32767,0.20412800709406534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,7,0.00772266648709774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,7,0.008890666688481966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,32767,0.15094932913780212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,15,0.007802666475375493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,15,0.008879999940594038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,31,0.00797333319981893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,31,0.00927466650803884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,63,0.008570666735370954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,63,0.009749333063761393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,127,0.008714666590094566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,127,0.01138666644692421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,255,0.00874133345981439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,255,0.011440000186363855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,511,0.01210133358836174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,511,0.009754666437705358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,1023,0.015365333606799444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,1023,0.01553600033124288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,2047,0.016656000167131424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,2047,0.0164533331990242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,4095,0.016538667182127636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,4095,0.0162773331006368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,8191,0.016672000288963318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,8191,0.016528000434239704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,65535,0.3839840094248454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,65535,0.27803200483322144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,16383,0.016890666137139004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,16383,0.016607999801635742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,32767,0.017605333278576534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,32767,0.016895999511082966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,1,0.007957333077987036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,1,0.009296000003814697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,3,0.008000000069538752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,3,0.00922133338948091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,65535,0.02256533255179723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,7,0.008053333188096682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,65535,0.021930667261282604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,15,0.008047999814152718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,7,0.00915733352303505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,15,0.009248000259200731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,31,0.00816000004609426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,31,0.009296000003814697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,63,0.008965333302815756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,63,0.010053333515922228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,127,0.009125333279371262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,127,0.011674666156371435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,255,0.009328000247478485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,255,0.01163200040658315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,131071,0.05772800246874491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,511,0.010415999839703241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,131071,0.0317493329445521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,511,0.011760000139474869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,131071,0.44012268384297687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,1023,0.02661866694688797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,1023,0.026127999027570088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,2047,0.03138133386770884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,2047,0.030261332790056866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,131071,0.7117546399434408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,4095,0.0531626691420873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,4095,0.041264000038305916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,8191,0.10770666599273682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,8191,0.06778666873772939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,16383,0.19215466578801474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,1,0.008672000219424566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,16383,0.12756266196568808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,1,0.008672000219424566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,3,0.00847999999920527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,7,0.008837333569924036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,3,0.008618666479984919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,7,0.00874133345981439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,15,0.008874666566650072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,15,0.008778666456540426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,32767,0.364954670270284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,31,0.009397333487868309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,31,0.009530666594703993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,32767,0.2299306591351827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,63,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,63,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,127,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,127,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,255,0.011584000041087469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,255,0.011349332829316458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,511,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,511,0.012986666212479273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,1023,0.01869333287080129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,1023,0.018181333939234417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,2047,0.05236800014972687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,2047,0.028768000503381092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,4095,0.09640000263849895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,4095,0.061861331264177956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,1,0.015365333606799444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,8191,0.18300267060597739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,65535,0.4365493456522624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,8191,0.11268799503644307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,1,0.014170666535695394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,3,0.014106666048367819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,3,0.014186666657527288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,7,0.014175999909639359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,7,0.014405333747466406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,15,0.014277332772811254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,15,0.014533333480358124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,65535,0.7115946610768636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,31,0.015578666081031164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,31,0.01544533297419548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,63,0.018581333259741466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,16383,0.3568586508433024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,63,0.01874133323629697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,16383,0.2138986587524414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,127,0.019061333189407986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,127,0.019018666197856266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,255,0.019088000059127808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,255,0.01887999971707662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,511,0.02213866760333379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,511,0.02342933416366577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,1023,0.04420800010363261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,1023,0.03410666684309641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,1,0.007877333089709282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,2047,0.09905067086219788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,1,0.00890666681031386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,2047,0.06057066718737284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,3,0.008874666566650072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,3,0.007850666840871176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,15,0.007706666365265846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,7,0.007738666608929634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,7,0.008890666688481966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,15,0.008943999807039896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,4095,0.11714133620262146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,31,0.007936000203092894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,4095,0.18469866116841635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,31,0.00915733352303505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,63,0.008565333361426989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,63,0.009935999910036722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,127,0.008767999708652496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,127,0.011440000186363855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,255,0.008943999807039896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,255,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,511,0.009872000043590864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,511,0.011525332927703857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,8191,0.21850132942199707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,8191,0.3556480010350545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,1023,0.015541333705186844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,1023,0.015589332828919092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,2047,0.01648533344268799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,2047,0.01629866659641266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,4095,0.016623999923467636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,4095,0.016229332735141117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,8191,0.016885332763195038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,8191,0.018160000443458557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,16383,0.02749866743882497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,16383,0.0272533322374026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,32767,0.03802666564782461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,32767,0.03751466671625773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,65535,0.07861333092053731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,1,0.013882666826248169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,65535,0.05738666653633118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,1,0.014170666535695394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,3,0.014010666559139887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,3,0.014106666048367819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,7,0.014298666268587112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,7,0.014639999717473984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,15,0.014789332946141561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,15,0.015125333021084467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,131071,0.14124799768129984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,131071,0.10959466298421223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,31,0.01801066721479098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,31,0.018239999810854595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,63,0.018170667191346485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,63,0.01838933303952217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,127,0.0183999997874101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,127,0.01883200059334437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,255,0.022730665902296703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,255,0.021909333765506744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,511,0.042992000778516136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,511,0.03399466723203659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,1023,0.09839999675750732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,1023,0.060138667623202004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,2047,0.11799466609954834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,2047,0.1845973332722982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,1,0.02384000023206075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,1,0.024383999407291412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,3,0.02404266595840454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,4095,0.21980800231297812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,4095,0.3555413484573364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,3,0.024288001159826916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,7,0.025258667767047882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,7,0.02483733246723811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,15,0.02569066733121872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,15,0.02611733227968216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,31,0.031541332602500916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,31,0.03200533241033554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,63,0.03175999969244003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,63,0.03234133372704188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,127,0.034272000193595886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,127,0.032602667808532715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,255,0.0484746644894282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,255,0.04091199984153112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,1,0.007818666597207388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,511,0.07703466713428497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,1,0.009125333279371262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,3,0.007840000092983246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,511,0.06579199930032094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,7,0.007760000104705493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,3,0.009045333291093508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,7,0.009109333157539368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,15,0.007781333600481351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,15,0.008821333448092142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,31,0.007936000203092894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,31,0.00922133338948091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,1023,0.18620266517003378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,1023,0.1104159951210022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,63,0.008687999720374743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,63,0.00961599995692571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,127,0.008879999940594038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,127,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,255,0.008901333436369896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,255,0.011343999455372492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,511,0.009743999689817429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,511,0.011429333438475927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,1023,0.016735999534527462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,1023,0.015647999942302704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,2047,0.016714667280515034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,2047,0.22554133335749307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,2047,0.016480000068744022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,8191,0.027669332921504974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,2047,0.35782933235168457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,4095,0.016704000532627106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,4095,0.016672000288963318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,8191,0.02771199991305669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,16383,0.03870933254559835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,16383,0.03742400060097376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,32767,0.0787306676308314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,1,0.008890666688481966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,32767,0.05693866809209188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,1,0.009056000038981438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,3,0.008757333581646284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,3,0.00891733355820179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,7,0.008714666590094566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,7,0.009050666665037474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,15,0.00895999992887179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,15,0.009066666786869368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,31,0.009648000200589498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,65535,0.13612266381581625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,31,0.00972800018886725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,65535,0.10738133390744527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,63,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,63,0.011333333949247995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,127,0.011333333949247995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,127,0.011328000575304031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,255,0.01166933278242747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,255,0.01138666644692421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,511,0.015781333049138386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,511,0.015317333241303762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,1023,0.016730666160583496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,1023,0.016650666793187458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,2047,0.028389332195123036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,2047,0.027301333844661713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,131071,0.24823466936747232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,4095,0.03812800099452337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,131071,0.18917866547902426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,4095,0.037130666275819145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,8191,0.07488533357779185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,8191,0.05547733108202616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,16383,0.12621866663297018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,16383,0.0999679962793986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,1,0.008458666503429413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,32767,0.17760000626246134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,1,0.008810666700204214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,32767,0.22777599096298218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,3,0.008602666358153025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,3,0.008736000085870424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,7,0.008613333106040955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,7,0.008736000085870424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,15,0.008896000062425932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,15,0.00898133342464765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,65535,0.43423465887705487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,31,0.009402666861812273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,65535,0.3371573289235433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,31,0.009599999835093817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,63,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,63,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,127,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,127,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,255,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,255,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,511,0.01534933348496755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,511,0.015141333142916361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,1023,0.0161920003592968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,1023,0.01600533351302147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,2047,0.016058667252461117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,2047,0.016010666886965435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,4095,0.016399999459584553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,4095,0.016069332758585613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,16383,0.016490666816631954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,8191,0.01639466608564059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,8191,0.01623999948302905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,131071,0.8486026922861735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,131071,0.6494239966074625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,16383,0.016538667182127636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,32767,0.016778666526079178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,32767,0.01666133354107539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,65535,0.017562666287024815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,1,0.00867733359336853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,65535,0.017370666066805523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,1,0.00877333308259646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,3,0.008602666358153025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,3,0.008874666566650072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,7,0.008645333349704742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,7,0.008890666688481966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,15,0.008714666590094566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,15,0.009029333169261614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,31,0.009488000224033991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,31,0.009679999823371569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,63,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,63,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,127,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,127,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,255,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,131071,0.023728000621000927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,255,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,511,0.015130666395028433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,131071,0.02239466706911723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,511,0.015226667126019796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,1023,0.01616000011563301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,1023,0.0161920003592968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,2047,0.016234666109085083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,2047,0.016437333077192307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,4095,0.016490666816631954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,4095,0.016538667182127636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,8191,0.016528000434239704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,32767,0.017301333447297413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,8191,0.016757333030303318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,16383,0.016629333297411602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,16383,0.016421332955360413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,32767,0.016821333517630894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,1,0.008965333302815756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,1,0.008933333059151968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,3,0.00892800030608972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,65535,0.022757334013779957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,3,0.009082666908701261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,65535,0.022261333962281544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,7,0.008746666833758354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,7,0.009045333291093508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,15,0.009162666896979014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,15,0.009354666496316591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,31,0.009888000165422758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,31,0.009786666681369146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,131071,0.057861333092053734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,63,0.011391999820868174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,63,0.011333333949247995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,127,0.011509332805871964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,131071,0.03176533430814743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,127,0.011509332805871964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,255,0.011434666812419891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,255,0.011546666423479715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,511,0.016106666376193363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,2047,0.03430933256944021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,511,0.016010666886965435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,1023,0.02762666592995326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,1023,0.027056001126766205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,2047,0.03571200122435888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,4095,0.06447466711203258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,4095,0.04855999847253164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,8191,0.11269866426785786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,8191,0.08687466382980347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,16383,0.20438400904337564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,16383,0.15531200170516968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,32767,0.28879467646280926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,32767,0.384549339612325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,1,0.008623999853928884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,1,0.00891733355820179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,3,0.00871999996403853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,3,0.008954666554927826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,7,0.008645333349704742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,7,0.008874666566650072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,15,0.008938666433095932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,15,0.009029333169261614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,31,0.009578666960199675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,65535,0.7423733075459799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,31,0.00980266680320104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,65535,0.5521706740061442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,63,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,63,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,127,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,127,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,255,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,255,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,511,0.015306666493415833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,511,0.01504533365368843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,1023,0.016399999459584553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,1023,0.016506666938463848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,2047,0.016586666305859882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,2047,0.01626666635274887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,4095,0.0164533331990242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,4095,0.016442666451136272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,8191,0.016480000068744022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,8191,0.01676799977819125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,16383,0.01721599946419398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,16383,0.016693333784739178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,32767,0.022384000321229298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,32767,0.021920000513394673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,131071,1.3943467140197754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,1,0.009082666908701261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,65535,0.05752533177534739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,1,0.009306666751702627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,65535,0.03204799940188726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,3,0.009093333035707474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,3,0.009322666873534521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,7,0.009093333035707474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,131071,0.8947679996490479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,7,0.009392000113924345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,15,0.009279999881982803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,15,0.009541333342591921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,31,0.0099093330403169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,31,0.010149333626031876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,131071,0.10116266210873921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,131071,0.06864533325036366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,63,0.011722666521867117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,63,0.01157333329319954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,127,0.01184533288081487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,127,0.011727999895811081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,255,0.012229333321253458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,255,0.011936000237862269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,511,0.02681066592534383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,511,0.025829332570234936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,1023,0.03109866629044215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,1023,0.03030933439731598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,2047,0.05246399839719137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,2047,0.04250133534272512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,4095,0.10730133454004924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,4095,0.07002666592597961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,8191,0.1917546590169271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,8191,0.13237866759300232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,16383,0.3616480032602946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,16383,0.2402026653289795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,1,0.008645333349704742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,1,0.00871999996403853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,3,0.008762666955590248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,3,0.008842666943868002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,7,0.008762666955590248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,7,0.008789333204428354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,15,0.009354666496316591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,15,0.009546666716535887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,32767,0.4568479855855306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,32767,0.7015199661254883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,31,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,31,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,63,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,63,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,127,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,127,0.011525332927703857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,255,0.01321600005030632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,255,0.013290667285521826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,511,0.018624000251293182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,511,0.018112000077962875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,1023,0.05293866495291392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,1023,0.02855466554562251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,2047,0.09585066636403401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,2047,0.061253334085146584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,4095,0.18212266763051352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,4095,0.113045334815979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,1,0.013845333208640417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,8191,0.35361599922180176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,8191,0.2152586579322815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,1,0.014266667266686758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,3,0.014533333480358124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,3,0.014240000396966934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,7,0.014159999787807465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,65535,0.8865546385447184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,16383,0.7059146563212076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,7,0.014458666245142618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,16383,0.42122666041056317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,15,0.015370666980743408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,15,0.015664000064134598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,65535,1.3901972770690918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,31,0.018661333868900936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,63,0.018858666221300762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,31,0.018613333503405254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,63,0.01868266612291336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,127,0.01904533306757609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,127,0.019695999721686046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,255,0.022389332453409832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,255,0.02366400013367335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,511,0.04375466704368591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,511,0.03410666684309641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,1023,0.060309335589408875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,1023,0.09943999846776326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,1,0.00867733359336853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,1,0.008837333569924036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,2047,0.18449600537618002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,2047,0.11707733074824016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,3,0.008565333361426989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,3,0.008853333070874214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,7,0.008901333436369896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,7,0.008805333326260248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,15,0.00891733355820179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,15,0.009029333169261614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,4095,0.21764800945917764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,31,0.009509333098928133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,31,0.00960533320903778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,4095,0.3552693525950114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,63,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,63,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,127,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,127,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,1023,0.01647466669480006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,255,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,255,0.01138666644692421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,511,0.015397333850463232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,511,0.015392000476519266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,4095,0.016719999412695568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,8191,0.6995946566263834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,8191,0.4212053219477336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,1023,0.01637866720557213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,2047,0.016501333564519882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,2047,0.016095999628305435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,4095,0.016575999557971954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,8191,0.027461332579453785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,8191,0.02741866558790207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,16383,0.03841600070397059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,16383,0.03736000011364619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,32767,0.08052800099054973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,32767,0.05801066756248474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,65535,0.11410666505495708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,65535,0.14139200250307718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,1,0.013770667215188345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,1,0.01402666668097178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,3,0.01402666668097178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,3,0.014533333480358124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,7,0.014730667074521383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,7,0.014890667051076889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,15,0.01791999985774358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,15,0.01811733345190684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,31,0.017968000223239262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,31,0.01807466646035512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,63,0.018191999445358913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,131071,0.26147733132044476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,63,0.018090666582187016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,131071,0.20476800203323364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,127,0.022970666488011677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,127,0.02145066608985265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,255,0.04260799785455068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,255,0.03373866776625315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,511,0.09815999865531921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,1023,0.11885333061218262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,1023,0.18447466691335043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,2047,0.3556160132090251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,2047,0.22216532627741495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,1,0.023754666248957317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,1,0.024319998919963837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,3,0.024383999407291412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,3,0.025248001019159954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,4095,0.4286400079727173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,4095,0.6988266309102377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,7,0.025333332518736523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,7,0.026074667771657307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,15,0.03153600047032038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,15,0.03200000027815501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,31,0.03193599979082743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,31,0.03182400017976761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,63,0.03427733232577642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,63,0.032255999743938446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,127,0.04804799954096476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,127,0.041093334555625916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,255,0.07782933115959167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,255,0.0664106657107671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,1,0.008725333337982496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,511,0.18713066975275675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,511,0.11120532949765523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,1,0.008943999807039896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,3,0.00878399983048439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,3,0.008672000219424566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,7,0.008762666955590248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,7,0.00898133342464765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,15,0.008933333059151968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,15,0.009194666519761086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,1023,0.2270666758219401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,31,0.009493333597977957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,1023,0.3582773208618164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,31,0.009872000043590864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,63,0.011312000453472137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,63,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,511,0.05982399980227152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,127,0.011338666081428528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,127,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,255,0.011551999797423681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,255,0.011434666812419891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,511,0.01540800059835116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,511,0.015589332828919092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,1023,0.01653333380818367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,1023,0.0169813334941864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,4095,0.027280000348885853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,2047,0.016634666671355564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,2047,0.01657066618402799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,2047,0.6997866630554199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,8191,0.037845333417256675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,4095,0.027477333943049114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,2047,0.43424534797668457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,8191,0.03821333249409994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,16383,0.07944533228874207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,16383,0.05853333572546641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,32767,0.13778133193651834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,32767,0.1125866671403249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,1,0.04218133290608724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,1,0.04278933505217234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,1,0.042378668983777366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,1,0.04204266766707102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,3,0.04344533383846283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,3,0.04261866708596548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,3,0.042463997999827065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,3,0.04215999941031138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,7,0.044106667240460716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,7,0.04569066564242045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,65535,0.19615999857584634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,7,0.04417066772778829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,65535,0.25033066670099896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,7,0.04372266431649526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,15,0.04539200166861216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,15,0.045423999428749084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,15,0.04528533418973287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,15,0.04555733501911163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,31,0.05605866511662801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,31,0.056090667843818665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,31,0.057573333382606506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,31,0.05707733333110809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,63,0.05693333347638448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,63,0.05699199934800466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,63,0.056832000613212585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,131071,0.3632693290710449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,131071,0.4770346482594808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,63,0.056847999493281044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,127,0.05842666824658712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,127,0.05766933163007101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,255,0.05735999842484792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,127,0.05775466561317444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,127,0.057760000228881836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,255,0.05749333401521047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,255,0.06900799771149953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,255,0.07049599786599477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,511,0.05791999896367391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,511,0.05737066765626272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,511,0.10688533385594685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,511,0.10778666536013286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,1023,0.05759466687838236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,1023,0.05778666834036509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,1023,0.18224533398946127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,1023,0.18519467115402222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,2047,0.05753066639105479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,2047,0.3343786795934041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,2047,0.05786666770776113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,2047,0.33795734246571857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,4095,0.058143998185793556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,4095,0.05750399827957153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,1,0.042064001162846885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,3,0.0423573354880015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,1,0.04254400233427683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,1,0.042437334855397545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,4095,0.6317600011825562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,4095,0.6425226529439291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,1,0.04260266820589701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,3,0.042117332418759666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,3,0.04246933261553446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,3,0.042581334710121155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,7,0.0443200021982193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,7,0.04377600053946177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,7,0.04394666850566864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,7,0.04419200122356415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,15,0.04524800181388855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,15,0.04557866851488749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,15,0.04600533346335093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,15,0.04721599817276001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,31,0.05690666536490122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,31,0.056847999493281044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,31,0.0576853354771932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,31,0.05701333284378052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,63,0.0576800008614858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,63,0.05769066512584686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,63,0.058005332946777344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,63,0.0581226646900177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,127,0.0581279993057251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,255,0.05849599838256836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,127,0.058133333921432495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,127,0.05841066439946493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,127,0.057999998331069946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,255,0.058703998724619545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,255,0.07070933282375336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,255,0.07057066758473714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,511,0.05845333139101664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,511,0.10844266414642334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,511,0.05796800057093302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,511,0.10938666264216106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,1023,0.0582826683918635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,1023,0.05840533475081126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,1023,0.18447999159495035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,1023,0.1878933310508728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,2047,0.05842666824658712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,2047,0.05845333139101664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,2047,0.33499733606974286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,4095,0.058330665032068886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,2047,0.3437173366546631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,4095,0.05850133299827576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,4095,0.6502666473388672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,4095,0.6422933340072632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,1,0.008330666770537695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,3,0.008287999778985977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,3,0.008538666491707167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,1,0.008042666440208754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,7,0.008618666479984919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,7,0.008496000121037165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,1,0.012240000069141388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,1,0.012096000214417776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,3,0.01201066623131434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,31,0.012047999848922094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,31,0.012026666353146235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,7,0.012149333953857422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,31,0.008234666660428047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,7,0.012250666817029318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,63,0.011936000237862269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,3,0.012085333466529846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,15,0.011941333611806234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,63,0.011733333269755045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,63,0.008074666683872541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,127,0.011685332904259363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,127,0.012042666474978128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,15,0.011898666620254517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,127,0.008896000062425932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,127,0.008879999940594038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,255,0.021770666042963665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,255,0.02149333308140437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,255,0.010357333347201347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,255,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,511,0.02733866622050603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,511,0.02992533395687739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,511,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,511,0.018789333601792652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,1023,0.03935466706752777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,1023,0.04804799954096476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,1023,0.011493333925803503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,1023,0.028325334191322327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,2047,0.04156800111134847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,2047,0.06239999830722809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,2047,0.011429333438475927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,2047,0.0365226666132609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,4095,0.041509332756201424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,4095,0.08029333253701527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,4095,0.011637333780527115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,4095,0.05575466652711233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,15,0.008709333216150602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,15,0.007989333321650824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,31,0.008597333605090777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,63,0.008378666515151659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,1,0.007226666435599327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,1,0.007349333415428798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,3,0.007226666435599327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,7,0.00730666642387708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,7,0.007237333183487256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,3,0.007402666533986728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,1,0.014970666418472925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,1,0.015066667149464289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,15,0.015130666395028433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,31,0.015274666249752045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,31,0.015205333630243937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,31,0.007370666911204656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,3,0.014858666807413101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,31,0.007407999907930692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,63,0.014837333311637243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,3,0.014970666418472925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,63,0.015040000279744467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,127,0.014837333311637243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,127,0.014848000059525171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,127,0.0080960001796484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,127,0.008047999814152718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,255,0.02722666660944621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,255,0.02717866748571396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,255,0.009765333185593287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,255,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,511,0.03692800054947535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,511,0.04144533226887385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,511,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,511,0.01918399954835574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,7,0.01525866612792015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,1023,0.03738133360942205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,1023,0.051141331593195595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,7,0.015018666783968607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,1023,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,1023,0.023887999355793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,2047,0.039120001097520195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,2047,0.06860266625881195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,2047,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,2047,0.03485333422819773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,4095,0.03933866570393244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,4095,0.09073600172996521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,4095,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,1,0.007967999825874964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,4095,0.05604266623655955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,1,0.007994666695594788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,1,0.009125333279371262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,1,0.009269333134094873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,3,0.007840000092983246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,3,0.007978666573762894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,3,0.009248000259200731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,3,0.009098666409651438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,7,0.007989333321650824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,7,0.008000000069538752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,7,0.009183999771873156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,15,0.015306666493415833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,7,0.009205333267649015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,15,0.00797333319981893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,15,0.007840000092983246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,15,0.009141333401203156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,15,0.009072000160813332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,31,0.008176000167926153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,31,0.008181333541870117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,31,0.009338666374484697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,31,0.009285333255926767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,63,0.008832000195980072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,63,0.008799999952316284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,63,0.009877333417534828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,63,0.009866666669646898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,127,0.008976000050703684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,15,0.007322666545708974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,15,0.007237333183487256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,127,0.008986666798591614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,127,0.011413333316644033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,127,0.011418666690587997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,255,0.009008000294367472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,255,0.009077333534757296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,255,0.011488000551859537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,255,0.011509332805871964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,511,0.009045333291093508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,511,0.009999999776482582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,511,0.011365332951148352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,63,0.007189333438873291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,511,0.01163200040658315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,63,0.007087999954819679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,1023,0.022015998760859173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,1023,0.0260959987839063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,1023,0.021984001000722248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,1023,0.02589333305756251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,2047,0.021957332889238994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,2047,0.030346666773160298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,2047,0.02205866575241089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,4095,0.021941334009170532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,2047,0.030224000414212544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,4095,0.04049066702524821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,8191,0.06061866879463196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,4095,0.02178666740655899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,4095,0.0402399996916453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,8191,0.021962667504946392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,8191,0.021941334009170532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,8191,0.06041066845258077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,16383,0.021935999393463135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,16383,0.10088533163070679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,16383,0.021850667893886566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,1,0.007925333455204964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,3,0.007962666451931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,1,0.00789866658548514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,16383,0.10001599788665771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,1,0.009018666421373686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,1,0.009119999905427298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,3,0.007840000092983246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,3,0.009119999905427298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,3,0.009253333633144697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,7,0.007877333089709282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,7,0.00790933333337307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,7,0.009050666665037474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,7,0.009130666653315226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,15,0.00789866658548514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,15,0.007941333577036858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,15,0.009178666397929192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,15,0.009072000160813332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,31,0.0080960001796484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,31,0.008181333541870117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,31,0.009216000015536943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,31,0.009226666763424873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,63,0.008826666822036108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,63,0.008976000050703684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,63,0.009941333283980688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,63,0.009872000043590864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,127,0.008954666554927826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,127,0.008842666943868002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,127,0.01146666705608368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,127,0.011482667177915573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,255,0.00902399979531765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,255,0.009056000038981438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,255,0.011413333316644033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,255,0.011600000162919363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,511,0.008976000050703684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,511,0.010005333150426546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,511,0.011450666934251785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,511,0.011610666910807291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,1023,0.02207999924818675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,1023,0.02608533451954524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,1023,0.02181333303451538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,1023,0.025802666942278545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,2047,0.021909333765506744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,4095,0.04055466751257578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,2047,0.030282666285832722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,2047,0.02216000109910965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,8191,0.021856000026067097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,2047,0.030405332644780476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,4095,0.022069332500298817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,4095,0.021946666141351063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,4095,0.04011733333269755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,8191,0.060458665092786155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,8191,0.021888000269730885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,8191,0.06055466830730438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,16383,0.02189333240191142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,16383,0.10068266590436299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,16383,0.021903999149799347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,1,0.007776000226537387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,1,0.007717333113153775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,16383,0.10008533795674641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,1,0.00789866658548514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,1,0.007941333577036858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,7,0.007920000081261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,3,0.00784533346692721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,3,0.008031999692320824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,3,0.00784533346692721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,3,0.007786666974425316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,7,0.007770666852593422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,7,0.007914666707317034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,7,0.00789866658548514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,15,0.007813333223263422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,15,0.007946666950980822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,15,0.007877333089709282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,15,0.008287999778985977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,31,0.008042666440208754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,31,0.007994666695594788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,31,0.008314666648705801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,31,0.007823999971151352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,63,0.008832000195980072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,127,0.009082666908701261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,63,0.008853333070874214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,63,0.007871999715765318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,63,0.008416000132759413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,127,0.008863999818762144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,127,0.008789333204428354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,127,0.008810666700204214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,255,0.008922666932145754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,255,0.00895999992887179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,255,0.009861333295702934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,255,0.010672000547250112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,511,0.00897066667675972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,1023,0.02610666553179423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,511,0.0099093330403169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,511,0.009722666814923286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,511,0.010549332946538925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,1023,0.021909333765506744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,1023,0.009973333527644476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,2047,0.013114667187134424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,1023,0.012042666474978128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,2047,0.013797332843144735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,2047,0.013823999712864557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,4095,0.01441066712141037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,2047,0.009269333134094873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,4095,0.01404800017674764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,4095,0.015119999647140503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,4095,0.009205333267649015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,8191,0.01492799942692121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,16383,0.015477333217859268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,16383,0.0194560003777345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,8191,0.018245333184798557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,16383,0.020741333564122517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,8191,0.00933333362142245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,8191,0.017551999539136887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,16383,0.00919999989370505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,1,0.007786666974425316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,1,0.007903999959429106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,1,0.006506666541099548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,1,0.006757333253820737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,3,0.007903999959429106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,3,0.007983999947706858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,3,0.0064853330453236895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,3,0.006671999891599019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,7,0.008421333506703377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,7,0.007871999715765318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,7,0.0069386667261521024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,7,0.006815999746322632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,15,0.00789866658548514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,15,0.007850666840871176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,15,0.006640000268816948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,15,0.006864000111818314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,63,0.00878399983048439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,31,0.008133333176374435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,31,0.008181333541870117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,31,0.006789333497484525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,31,0.006533333410819371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,63,0.008746666833758354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,63,0.006895999734600385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,63,0.006751999879876773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,127,0.008943999807039896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,127,0.009088000282645226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,127,0.0074986666440963745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,127,0.007333333293596904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,255,0.00895999992887179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,255,0.009088000282645226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,255,0.008069333309928576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,255,0.008954666554927826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,511,0.009119999905427298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,511,0.00997866690158844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,511,0.008005333443482717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,511,0.00926399976015091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,1023,0.012400000045696894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,1023,0.008037333066264788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,1023,0.009882666791478792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,1023,0.012293333808581034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,2047,0.012304000556468964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,2047,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,2047,0.007989333321650824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,2047,0.010687999427318573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,4095,0.012901333471139273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,4095,0.013151999562978745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,4095,0.008154666672150293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,4095,0.012186666329701742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,8191,0.01580799991885821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,8191,0.020106667031844456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,8191,0.0080960001796484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,1,0.00860799973209699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,1,0.008661333471536636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,8191,0.01658133293191592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,16383,0.016800000021855038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,16383,0.024160000185171764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,16383,0.008693333094318708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,3,0.008453333129485449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,16383,0.019802667200565338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,1,0.008506666868925095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,1,0.008581333483258883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,7,0.008576000109314919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,3,0.008592000231146812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,3,0.008714666590094566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,3,0.008736000085870424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,7,0.008767999708652496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,7,0.008933333059151968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,7,0.0086666668454806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,15,0.00877333308259646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,15,0.008656000097592672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,15,0.008639999975760778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,15,0.008767999708652496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,31,0.009541333342591921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,31,0.009423999736706415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,31,0.009466666728258133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,31,0.009434666484594345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,63,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,63,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,63,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,63,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,127,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,127,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,127,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,127,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,255,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,255,0.011328000575304031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,255,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,255,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,511,0.01126933346192042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,511,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,511,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,511,0.012778667112191519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,1023,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,2047,0.0277813325325648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,1023,0.017898666361967724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,1023,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,1023,0.01814933369557063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,2047,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,2047,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,2047,0.028762665887673695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,4095,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,4095,0.04740799963474274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,4095,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,4095,0.04979733129342397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,8191,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,8191,0.08703999718030293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,8191,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,8191,0.09110400080680847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,1,0.008592000231146812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,16383,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,16383,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,16383,0.16684265931447348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,1,0.008714666590094566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,16383,0.17479999860127768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,1,0.008463999877373377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,1,0.008623999853928884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,3,0.008597333605090777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,3,0.008703999842206636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,3,0.008618666479984919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,3,0.00867733359336853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,7,0.008725333337982496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,7,0.00860799973209699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,7,0.00879466657837232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,7,0.008687999720374743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,15,0.008901333436369896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,31,0.009328000247478485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,15,0.00879466657837232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,15,0.008693333094318708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,15,0.0086666668454806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,31,0.00949866697192192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,31,0.009482666850090027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,31,0.009408000235756239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,63,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,127,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,63,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,63,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,63,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,127,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,127,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,127,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,255,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,255,0.011322667201360067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,255,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,255,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,511,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,511,0.012991999586423239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,511,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,511,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,2047,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,1023,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,1023,0.018031999468803406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,1023,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,1023,0.018053332964579265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,2047,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,2047,0.028117333849271137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,2047,0.02867199977238973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,4095,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,4095,0.04762666424115499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,4095,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,16383,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,4095,0.050010666251182556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,8191,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,8191,0.08739733695983887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,8191,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,8191,0.09099733829498291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,16383,0.16561599572499594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,16383,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,1,0.008613333106040955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,16383,0.17384000619252524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,1,0.008778666456540426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,1,0.007834666719039282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,1,0.007978666573762894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,3,0.008576000109314919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,3,0.008485333373149237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,3,0.007770666852593422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,3,0.007882666463653246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,7,0.008890666688481966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,7,0.00873066671192646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,7,0.008021333565314611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,7,0.007936000203092894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,15,0.008661333471536636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,15,0.00871999996403853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,15,0.008303999900817871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,15,0.008394666636983553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,31,0.009408000235756239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,31,0.00949866697192192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,31,0.007989333321650824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,31,0.008223999912540117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,63,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,63,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,63,0.008389333263039589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,63,0.008223999912540117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,127,0.011365332951148352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,127,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,127,0.009082666908701261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,127,0.009173333023985228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,255,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,255,0.011461333682139715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,255,0.009626666704813639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,255,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,511,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,511,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,511,0.009882666791478792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,511,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,1023,0.013594667116800943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,1023,0.013455999394257864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,1023,0.009472000102202097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,1023,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,2047,0.013973332941532135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,4095,0.009349333122372627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,2047,0.009306666751702627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,2047,0.014853333433469137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,2047,0.012746666868527731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,4095,0.014752000570297241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,4095,0.015530666957298914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,4095,0.01443733274936676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,8191,0.01821333294113477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,8191,0.02401600033044815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,8191,0.009663999701539675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,8191,0.02070933332045873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,16383,0.01934933289885521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,16383,0.02865600089232127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,16383,0.00984533317387104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,1,0.008522666369875273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,16383,0.02531733363866806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,1,0.008703999842206636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,1,0.006586666529377301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,1,0.006480000292261441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,3,0.00855466661353906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,3,0.008810666700204214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,3,0.006560000280539195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,3,0.006735999758044879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,7,0.008799999952316284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,7,0.008885333314538002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,7,0.00684799998998642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,7,0.006784000123540561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,15,0.008634666601816813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,15,0.00884799969693025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,15,0.00660800002515316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,15,0.006778666749596596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,31,0.009509333098928133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,31,0.009541333342591921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,31,0.006864000111818314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,31,0.006730666384100914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,63,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,63,0.011653333902359009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,63,0.006810666372378667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,63,0.006677333265542984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,127,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,127,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,127,0.007680000116427739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,127,0.007280000175038974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,255,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,255,0.011429333438475927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,255,0.008016000191370646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,255,0.008826666822036108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,1023,0.008042666440208754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,511,0.011658667276302973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,511,0.011936000237862269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,511,0.00814933329820633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,511,0.009162666896979014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,1023,0.011717333147923151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,1023,0.012181332955757776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,1023,0.009541333342591921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,2047,0.015429332852363586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,2047,0.01781333362062772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,2047,0.008378666515151659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,2047,0.01191466674208641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,8191,0.027786667148272198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,4095,0.015925332903862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,4095,0.019626667102177937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,4095,0.008517333616813024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,4095,0.013786666095256805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,8191,0.019440000255902607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,8191,0.008192000289758047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,8191,0.01877333347996076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,16383,0.022453332940737408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,16383,0.03774400055408478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,16383,0.008357333640257517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,16383,0.025311999022960663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,1,0.07877866427103679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,1,0.0788266658782959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,1,0.07884266475836436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,3,0.07898666461308797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,1,0.07943999767303467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,3,0.07829866806666057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,7,0.08220799763997395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,3,0.0788213312625885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,3,0.07941866914431255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,7,0.08230933547019958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,7,0.08188266555468242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,15,0.08540266752243042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,7,0.0828000009059906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,15,0.0843999981880188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,15,0.08447466293970744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,15,0.08542399605115254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,31,0.10761066277821858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,31,0.10698133707046509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,63,0.10885333021481831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,31,0.10849600036938985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,31,0.10863999525705974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,127,0.10936533411343892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,63,0.10753066341082256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,63,0.10763200124104817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,63,0.10900266965230306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,127,0.11008532842000325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,255,0.10980266332626343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,127,0.10992532968521118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,127,0.11044800281524658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,255,0.13219199577967325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,255,0.10930132865905762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,255,0.13300800323486328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,511,0.11109333237012227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,511,0.20576000213623047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,511,0.10993599891662598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,511,0.20869332551956177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,1023,0.10985066493352254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,1023,0.11013333002726237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,1023,0.35354665915171307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,1023,0.3602293332417806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,2047,0.11008532842000325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,2047,0.11010666688283284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,1,0.07950399816036224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,1,0.07884799937407176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,2047,0.6509173313776652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,1,0.07969599962234497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,2047,0.6561439832051595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,1,0.0794293334086736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,3,0.0794239987929662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,3,0.07950933277606964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,3,0.07941333452860515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,3,0.07987200220425923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,7,0.08261866867542267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,7,0.08212266862392426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,7,0.08258666594823201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,7,0.08295466502507527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,31,0.1076746682325999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,15,0.08567466338475545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,15,0.08559999863306682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,15,0.0853653351465861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,15,0.08588266372680664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,31,0.10854933659235637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,31,0.10841066638628642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,31,0.10911466677983601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,63,0.10853333274523418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,63,0.10956799983978271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,63,0.10998933513959248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,63,0.10922666390736897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,127,0.11097066601117452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,127,0.11100799838701884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,127,0.11040000120798747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,127,0.11086933811505635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,255,0.1106719970703125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,255,0.13492266337076822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,255,0.11108266313870747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,255,0.13316800196965536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,511,0.11204266548156738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,511,0.2090453306833903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,511,0.11040000120798747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,511,0.21077867348988852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,1023,0.11117333173751831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,1023,0.11109866698582967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,1023,0.3565760056177775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,1023,0.3606826861699422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,2047,0.11183999975522359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,2047,0.11029866337776184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,2047,0.657530665397644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,1,0.017866666118303936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,1,0.017642666896184284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,1,0.008810666700204214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,2047,0.6627466678619385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,3,0.008656000097592672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,1,0.008816000074148178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,3,0.017797333498795826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,7,0.017594666530688603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,3,0.017397332936525345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,3,0.008725333337982496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,7,0.017637333522240322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,7,0.008703999842206636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,7,0.00877333308259646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,15,0.01786133274435997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,15,0.01764800027012825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,15,0.008549333239595095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,15,0.008858666444818178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,31,0.017616000026464462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,31,0.017605333278576534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,31,0.008618666479984919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,31,0.008661333471536636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,63,0.017456000049908955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,63,0.01730666682124138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,63,0.008879999940594038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,63,0.008592000231146812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,127,0.01746133342385292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,127,0.017423999806245167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,127,0.009626666704813639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,127,0.009509333098928133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,255,0.03257599969704946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,255,0.03289066751797994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,255,0.016751999656359356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,255,0.01903466631968816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,511,0.04363733530044556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,511,0.04934933284918467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,511,0.017152000218629837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,511,0.027850667635599773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,1023,0.044064000248909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,1023,0.06128533184528351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,1023,0.01732800031701724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,2047,0.017221332838137943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,1023,0.035242666800816856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,2047,0.046154667933781944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,2047,0.08264000217119853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,2047,0.05485333502292633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,1,0.023962666591008503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,3,0.024325333535671234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,3,0.02404800057411194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,1,0.023872000475724537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,1,0.007903999959429106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,1,0.007871999715765318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,3,0.008005333443482717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,3,0.00797333319981893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,7,0.02425066630045573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,7,0.024186665813128155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,7,0.00790933333337307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,7,0.00786666696270307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,31,0.02420799930890401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,15,0.023936000963052113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,15,0.02399466683467229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,15,0.008005333443482717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,15,0.008016000191370646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,31,0.024197332561016083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,31,0.007941333577036858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,31,0.00790933333337307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,63,0.023738667368888855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,63,0.023658665517965954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,127,0.009343999748428663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,63,0.00790933333337307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,63,0.00797333319981893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,127,0.023823998868465424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,127,0.023754666248957317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,127,0.00960533320903778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,255,0.044362664222717285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,255,0.04409599800904592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,255,0.01575999955336253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,255,0.018650667121013004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,511,0.044256001710891724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,511,0.05454400181770325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,511,0.015802666544914246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,511,0.02385066697994868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,1023,0.044256001710891724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,1023,0.06739200154940288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,1023,0.015696000307798386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,1023,0.03514133393764496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,2047,0.046762665112813316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,2047,0.0932426651318868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,1,0.013525333255529404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,2047,0.01581866666674614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,2047,0.05628266433874766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,1,0.013717333475748697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,1,0.013701333353916803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,1,0.013722666849692663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,3,0.013877333452304205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,3,0.013712000101804733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,3,0.013679999858140945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,3,0.013642666240533194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,7,0.014053333550691605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,7,0.013983999689420065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,7,0.013765333841244379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,7,0.013936000565687815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,15,0.014127999544143677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,15,0.013909333695967993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,15,0.01379199946920077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,15,0.013872000078360239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,31,0.015333333363135656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,31,0.01516266663869222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,63,0.018453333526849747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,31,0.01509333277742068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,31,0.01534933348496755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,63,0.01852799952030182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,63,0.018453333526849747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,63,0.018559999763965607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,127,0.018687999496857326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,127,0.018629333625237148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,127,0.018485333770513535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,127,0.018757333358128864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,255,0.018789333601792652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,255,0.018992000569899876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,255,0.018853332847356796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,255,0.018730666488409042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,511,0.018735999862353008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,511,0.021888000269730885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,511,0.018800000349680584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,511,0.02184533327817917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,1023,0.018794666975736618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,1023,0.03206400076548258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,1023,0.018624000251293182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,1023,0.03165333221356074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,2047,0.018613333503405254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,2047,0.05153599878152212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,2047,0.018602666755517323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,4095,0.01858666663368543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,2047,0.05206400156021118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,4095,0.018965333700180054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,4095,0.09020800391832988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,4095,0.09134399890899658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,8191,0.019018666197856266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,8191,0.16820265849431357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,8191,0.018565333137909572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,8191,0.17097600301106772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,16383,0.018810667097568512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,16383,0.018474667022625606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,16383,0.32920000950495404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,1,0.013786666095256805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,1,0.013717333475748697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,1,0.013776000589132309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,16383,0.3243146737416585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,1,0.013776000589132309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,3,0.013893333574136099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,3,0.013701333353916803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,3,0.01394133393963178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,3,0.01392000044385592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,7,0.013893333574136099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,7,0.014671999961137772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,7,0.013936000565687815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,7,0.014053333550691605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,15,0.014127999544143677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,15,0.013893333574136099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,31,0.015247999380032221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,15,0.01423466702302297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,15,0.014117332796255747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,31,0.015205333630243937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,31,0.015119999647140503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,31,0.015114666273196539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,63,0.01848000039656957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,63,0.018325333793958027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,63,0.01855466639002164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,63,0.01868266612291336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,127,0.018719999740521114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,127,0.018570666511853535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,127,0.01846933364868164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,127,0.018613333503405254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,511,0.018506667266289394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,255,0.01889066646496455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,255,0.01876266673207283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,255,0.018789333601792652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,255,0.018858666221300762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,511,0.018863999595244724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,511,0.022069332500298817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,511,0.02183466653029124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,1023,0.018559999763965607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,1023,0.0322773332397143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,1023,0.01852799952030182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,1023,0.03206400076548258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,2047,0.018992000569899876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,2047,0.05116266508897146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,2047,0.018629333625237148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,2047,0.05189333359400431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,4095,0.018730666488409042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,4095,0.09059199690818787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,4095,0.018592000007629395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,4095,0.09357333183288574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,8191,0.018922666708628338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,8191,0.1707893411318461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,8191,0.018602666755517323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,16383,0.01850133389234543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,8191,0.16715200742085776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,16383,0.018757333358128864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,1,0.013989333063364029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,1,0.013717333475748697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,16383,0.3245813250541687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,1,0.007743999982873599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,1,0.007840000092983246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,3,0.013872000078360239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,16383,0.3288106719652812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,3,0.013679999858140945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,3,0.00790933333337307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,3,0.008245333408315977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,7,0.013978666315476099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,7,0.014005333185195923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,7,0.00795199970404307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,7,0.007920000081261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,15,0.014042666802803675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,15,0.01422400027513504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,15,0.008256000156203905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,15,0.008229333286484083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,31,0.015263999501864115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,31,0.015306666493415833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,31,0.007840000092983246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,31,0.008234666660428047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,63,0.018624000251293182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,63,0.01850133389234543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,63,0.008186666915814081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,63,0.008469333251317343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,127,0.01874133323629697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,127,0.0186666672428449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,127,0.009301333377758661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,127,0.008853333070874214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,255,0.01877333347996076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,255,0.01886933296918869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,255,0.009946666657924652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,255,0.010309333602587381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,511,0.01313599944114685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,511,0.013232000172138214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,511,0.009503999724984169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,511,0.010682666053374609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,1023,0.013248000293970108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,1023,0.013754667093356451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,1023,0.009434666484594345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,1023,0.011328000575304031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,4095,0.018944000204404194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,2047,0.018320000420014065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,2047,0.020975999534130096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,2047,0.00972800018886725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,2047,0.014533333480358124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,4095,0.023391999304294586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,4095,0.009749333063761393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,4095,0.017637333522240322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,8191,0.0227360005180041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,8191,0.033514666060606636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,8191,0.009984000275532404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,8191,0.024485332270463307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,16383,0.026565333207448322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,1,0.006618666773041089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,16383,0.04643199841181437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,16383,0.009957333405812582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,16383,0.04044266790151596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,1,0.01394133393963178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,1,0.013786666095256805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,1,0.006618666773041089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,3,0.013957332819700241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,3,0.013807999591032663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,3,0.006624000146985054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,3,0.006629333520929019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,7,0.014170666535695394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,15,0.0068853336075941724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,7,0.014058666924635569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,7,0.006655999769767125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,7,0.0068693334857622785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,15,0.013973332941532135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,15,0.006671999891599019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,15,0.014111999422311783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,31,0.015344000111023584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,31,0.01524266724785169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,31,0.006895999734600385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,31,0.006650666395823161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,63,0.018565333137909572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,63,0.018458666900793713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,63,0.006890666360656421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,63,0.006682666639486949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,127,0.01870399961868922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,127,0.018640000373125076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,127,0.007418666655818622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,127,0.007258666679263115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,255,0.011909333368142446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,255,0.011653333902359009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,255,0.008245333408315977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,255,0.008602666358153025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,511,0.011999999483426413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,511,0.012159999459981918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,511,0.00808533343176047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,511,0.008863999818762144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,1023,0.016719999412695568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,1023,0.01738133281469345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,1023,0.008522666369875273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,1023,0.010618666807810465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,2047,0.01931200052301089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,2047,0.024122667809327442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,2047,0.008858666444818178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,2047,0.01320533330241839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,4095,0.02380266785621643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,4095,0.03275733441114426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,4095,0.008512000242869059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,8191,0.02773866554101308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,4095,0.01777600000301997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,8191,0.028042666614055634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,8191,0.044682666659355164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,8191,0.00874133345981439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,16383,0.03397866586844126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,16383,0.008938666433095932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,16383,0.06909866631031036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,16383,0.039274667700131737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,float16,1,0.1514079968134562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,1,0.15003732840220133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,fp8,1,0.15249066551526388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,1,0.1525920033454895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,float16,3,0.15146666765213013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,float16,7,0.157642662525177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,3,0.15029866496721903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,fp8,3,0.15261866648991904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,3,0.15281066298484802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,7,0.15934933225313822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,fp8,7,0.15965333580970764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,7,0.15993600090344748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,float16,15,0.16331199804941812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,15,0.16325866182645163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,fp8,15,0.16539733608563742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,15,0.16503999630610147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,float16,31,0.20980799198150635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,31,0.20969067017237344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,fp8,31,0.2116373380025228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,31,0.20968000094095865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,float16,63,0.2095200022061666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,63,0.21122666200002035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,fp8,63,0.21284266312917074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,63,0.21237333615620932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,float16,127,0.21461333831151327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,127,0.2146986722946167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,fp8,127,0.21285333236058554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,fp8,255,0.21471999088923135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,127,0.2128373384475708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,float16,255,0.2152000069618225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,255,0.26080532868703205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,255,0.2600693305333455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,float16,511,0.21637332439422607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,fp8,511,0.2144320011138916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,511,0.4039253393809001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,float16,1023,0.21648534138997397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,511,0.40746132532755536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,fp8,1023,0.2146186629931132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,1023,0.7019680341084799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,fp8,1,0.15401599804560342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,float16,1,0.15289066235224405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,1,0.152837336063385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,1,0.15157866477966309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,1023,0.7120426495869955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,float16,3,0.153029332558314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,fp8,3,0.15413332978884378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,3,0.15187733372052512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,3,0.15431466698646545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,float16,7,0.15929067134857178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,7,0.158053328593572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,fp8,7,0.15892799695332846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,7,0.16029333074887595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,float16,15,0.16518400112787882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,31,0.20918933550516763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,15,0.16516266266504923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,fp8,15,0.16486400365829468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,15,0.16613866885503134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,float16,63,0.2125493288040161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,float16,31,0.21100799242655435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,fp8,31,0.21055465936660767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,31,0.21234132846196493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,63,0.21107200781504312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,fp8,63,0.21373866001764932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,63,0.2122933268547058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,float16,127,0.21640000740687051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,127,0.21618133783340454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,fp8,127,0.21610132853190103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,127,0.2144320011138916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,float16,255,0.2172000010808309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,255,0.26154132684071857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,fp8,255,0.21649599075317383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,255,0.2601333260536194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,float16,511,0.21820799509684244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,511,0.40963200728098553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,fp8,511,0.21617066860198975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,float16,1023,0.21828800439834595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,511,0.4100319941838582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,float16,1,0.029706666866938274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,fp8,1023,0.2164319952329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,1023,0.7037226359049479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,1,0.029839999973773956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,1023,0.7091360092163086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,fp8,1,0.014463999619086584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,1,0.014581333845853806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,float16,7,0.029711998999118805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,7,0.029152000943819683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,fp8,7,0.014565333724021912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,float16,3,0.029322666426499683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,3,0.029637334247430164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,fp8,3,0.014362666755914688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,3,0.014378666877746582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,7,0.014570667097965876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,15,0.014357333381970724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,float16,15,0.02951466788848241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,15,0.02932800104220708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,fp8,15,0.014479999740918478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,31,0.014416000495354334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,float16,31,0.029626667499542236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,31,0.029194665451844532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,fp8,31,0.01452800010641416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,float16,63,0.029146666328112285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,63,0.02903466671705246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,fp8,63,0.014362666755914688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,63,0.014266667266686758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,float16,127,0.029306667546431225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,127,0.029152000943819683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,fp8,127,0.01647466669480006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,127,0.016389333953460056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,float16,255,0.05427733560403188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,255,0.05455466608206431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,fp8,255,0.024485332270463307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,255,0.02794133375088374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,fp8,511,0.02446399877468745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,float16,511,0.0544106662273407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,511,0.06664533416430156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,511,0.03557866563399633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,float16,1023,0.05477866530418396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,1023,0.0831520011027654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,fp8,1023,0.024266667664051056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,1023,0.056405335664749146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,float16,1,0.04177600145339966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,1,0.042463997999827065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,fp8,1,0.012661332885424295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,1,0.01267733300725619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,float16,3,0.04164800047874451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,3,0.04173333446184794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,fp8,3,0.012576000144084295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,3,0.012661332885424295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,float16,7,0.04223999877770742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,7,0.04168533285458883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,fp8,7,0.012714666624863943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,7,0.012773333738247553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,float16,15,0.041589332123597465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,15,0.04186133543650309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,fp8,15,0.01268799975514412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,15,0.012784000486135483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,float16,31,0.041696002086003624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,63,0.040906667709350586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,31,0.04189866781234741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,fp8,31,0.01258133351802826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,31,0.012586666891972223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,float16,63,0.04107200105985006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,fp8,63,0.012698666503032049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,63,0.012618667135636011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,float16,127,0.040949332217375435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,127,0.041082667807737984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,fp8,127,0.015306666493415833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,127,0.01533866673707962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,float16,255,0.04457066456476847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,255,0.044154668847719826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,fp8,255,0.015333333363135656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,255,0.020469332734743755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,float16,511,0.04456000030040741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,511,0.057018667459487915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,fp8,511,0.01532799998919169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,511,0.032186667124430336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,float16,1023,0.044346665342648826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,1023,0.07989866534868877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,fp8,1023,0.01522133375207583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,1023,0.052943999568621315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,float16,1,0.29498666524887085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,float16,1,0.29532267649968463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,fp8,1,0.2978079915046692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,fp8,1,0.29789332548777264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,float16,3,0.29547733068466187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,float16,3,0.29338133335113525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,float16,7,0.31163734197616577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,fp8,3,0.29602134227752686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,fp8,3,0.2962239980697632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,float16,7,0.3116319974263509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,float16,15,0.3232213258743286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,fp8,7,0.31354133288065594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,fp8,7,0.3133760094642639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,float16,15,0.32080533107121784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,fp8,15,0.3248959978421529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,fp8,15,0.3228213389714559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,float16,31,0.41118399302164715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,float16,31,0.414192001024882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,fp8,31,0.41708799203236896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,fp8,31,0.41708799203236896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,float16,63,0.41656001408894855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,float16,63,0.41363199551900226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,fp8,63,0.41914133230845135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,fp8,63,0.4162026643753052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,float16,127,0.4208960135777791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,float16,127,0.4240693251291911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,fp8,127,0.42342933019002277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,fp8,127,0.42315200964609784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,float16,255,0.424885352452596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,float16,255,0.5109119812647501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,fp8,255,0.42368535200754803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,float16,1,0.2976693312327067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,float16,1,0.2999146580696106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,fp8,255,0.5103946526845297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,fp8,1,0.3022773265838623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,fp8,1,0.30209600925445557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,float16,3,0.298250675201416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,float16,7,0.3100693424542745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,float16,3,0.3003466725349426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,fp8,3,0.3002506693204244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,fp8,3,0.3027199904123942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,float16,7,0.3123520016670227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,fp8,7,0.31437333424886066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,fp8,7,0.312117338180542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,float16,15,0.322432001431783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,float16,15,0.32469866673151654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,fp8,15,0.32382933298746747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,fp8,15,0.3261813322703044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,fp8,31,0.4152373472849528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,fp8,31,0.4153013229370117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,float16,31,0.41576000054677326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,float16,31,0.41562668482462567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,float16,63,0.41916799545288086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,fp8,63,0.42136534055074054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,fp8,63,0.41844268639882404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,float16,127,0.42366401354471844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,float16,63,0.4161813259124756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,float16,127,0.4267786741256714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,fp8,127,0.42636267344156903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,fp8,127,0.4264800151189168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,float16,255,0.42867199579874676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,float16,1,0.05316799879074097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,fp8,255,0.4244053363800049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,float16,255,0.5196533203125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,float16,1,0.05273599922657013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,fp8,1,0.020394666741291683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,fp8,255,0.5140426556269327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,fp8,1,0.020341333001852036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,float16,3,0.05319466690222422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,float16,7,0.052522664268811546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,float16,7,0.052042668064435325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,float16,3,0.05220800141493479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,fp8,3,0.02012266715367635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,fp8,3,0.02027200038234393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,fp8,7,0.020367999871571858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,fp8,7,0.020341333001852036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,float16,15,0.05263466636339823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,float16,15,0.05208533505598704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,fp8,15,0.02027733375628789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,fp8,15,0.020106667031844456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,float16,31,0.052746668457984924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,float16,31,0.052469333012898765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,fp8,31,0.02032533288002014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,fp8,31,0.02035733312368393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,float16,63,0.051498666405677795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,float16,63,0.051541333397229515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,fp8,63,0.020080000162124634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,fp8,63,0.02021866664290428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,float16,127,0.05187733471393585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,float16,127,0.05188799897829691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,fp8,127,0.024122667809327442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,fp8,127,0.024138666689395905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,float16,255,0.0551146666208903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,float16,255,0.05412800113360087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,fp8,255,0.023957334458827972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,fp8,255,0.030933332939942677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,float16,1,0.0765066643555959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,float16,1,0.07638399799664815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,fp8,1,0.022085333863894146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,fp8,1,0.022218666970729828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,float16,3,0.07607466479142506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,float16,3,0.07595199843247731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,fp8,3,0.021829334398110706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,fp8,3,0.022143999735514324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,float16,7,0.0767626663049062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,float16,7,0.07637866834799449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,fp8,7,0.0220320001244545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,fp8,7,0.02223466585079829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,float16,15,0.07614399989446004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,float16,15,0.07603199779987335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,fp8,15,0.021776000658671062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,fp8,15,0.022122666239738464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,float16,31,0.07662400106589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,float16,31,0.07627200086911519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,fp8,31,0.02214933435122172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,fp8,31,0.022101332743962605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,float16,63,0.07483200232187907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,float16,127,0.07684266567230225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,float16,63,0.0746613343556722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,fp8,63,0.021642667551835377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,fp8,127,0.026949333647886913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,fp8,63,0.021920000513394673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,float16,127,0.07634666562080383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,fp8,127,0.0271573339899381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,float16,255,0.08175466458002727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,float16,255,0.08175999919573466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,fp8,255,0.026741333305835724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,fp8,255,0.04048533240954081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,float16,1,0.581717332204183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,float16,1,0.5818933248519897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,fp8,1,0.5874933401743571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,fp8,1,0.5875999927520752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,float16,3,0.5774293343226115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,float16,3,0.5818293491999308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,fp8,3,0.5836319923400879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,fp8,3,0.5878613392512003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,float16,7,0.6170506477355957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,float16,7,0.6170239845911661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,fp8,7,0.6214133501052856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,fp8,7,0.6171679894129435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,float16,15,0.640496015548706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,float16,15,0.636031985282898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,fp8,15,0.6443466742833456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,fp8,15,0.6443466742833456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,float16,31,0.8230933348337809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,float16,31,0.8233173688252767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,fp8,31,0.8230453332265218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,fp8,31,0.8291093508402506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,float16,63,0.8224586645762125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,float16,63,0.8283519744873047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,fp8,63,0.8334773381551107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,fp8,63,0.8334453105926514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,float16,127,0.8463679949442545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,float16,127,0.837994654973348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,fp8,127,0.8414666652679443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,fp8,127,0.8358346621195475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,float16,1,0.5984373490015665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,float16,1,0.5938133398691813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,fp8,1,0.5985120137532552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,fp8,1,0.5983680089314779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,float16,3,0.5906293392181396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,float16,3,0.5949653387069702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,fp8,3,0.5951679944992065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,fp8,3,0.5991466840108236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,float16,7,0.6182186603546143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,float16,7,0.6181813478469849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,fp8,7,0.6229759852091471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,fp8,7,0.618282675743103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,float16,15,0.643610676129659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,float16,15,0.639141321182251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,fp8,15,0.6469493309656779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,fp8,15,0.6466559966405233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,float16,31,0.8255840142567953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,float16,31,0.8268746534983317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,fp8,31,0.8248693148295084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,fp8,31,0.8249706427256266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,float16,63,0.8341653347015381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,float16,63,0.829535961151123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,fp8,63,0.8373066584269205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,fp8,63,0.8371520042419434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,float16,127,0.8634506861368815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,float16,127,0.8631093502044678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,float16,1,0.09673066933949788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,fp8,127,0.8431626955668131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,float16,1,0.09596799810727437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,fp8,127,0.8439093430836996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,fp8,1,0.03602133442958196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,fp8,1,0.036559998989105225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,float16,3,0.09703999757766724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,float16,3,0.09643200039863586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,fp8,3,0.03700799991687139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,fp8,3,0.03718933214743932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,float16,7,0.09569066762924194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,float16,7,0.0958720048268636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,fp8,15,0.03718933214743932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,fp8,7,0.03633599976698557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,fp8,7,0.03668266783157984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,float16,15,0.09738133351008098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,float16,15,0.09666666388511658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,fp8,15,0.03703466554482778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,float16,31,0.09630933403968811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,float16,31,0.09659199913342793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,fp8,31,0.03737599899371465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,fp8,63,0.03714133302370707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,float16,127,0.09618133306503296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,fp8,31,0.03624533365170161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,float16,63,0.09685333569844563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,float16,63,0.09642666578292847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,fp8,63,0.03663466622432073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,float16,127,0.09662933150927226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,fp8,127,0.04683733483155569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,fp8,127,0.046469335754712425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,float16,1,0.14170133074124655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,float16,1,0.1423679987589518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,fp8,1,0.04268266757329305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,fp8,1,0.04391466577847799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,float16,3,0.14283200105031332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,float16,3,0.14190399646759033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,fp8,3,0.042624001701672874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,fp8,3,0.04366933306058248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,float16,7,0.14169599612553915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,float16,7,0.14257066448529562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,fp8,7,0.04385066529115041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,fp8,7,0.04399466514587402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,float16,15,0.14362667004267374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,float16,15,0.14243200421333313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,fp8,15,0.042693331837654114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,fp8,15,0.043509334325790405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,float16,31,0.14353066682815552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,float16,63,0.14325333635012308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,float16,31,0.14427733421325684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,fp8,31,0.04285866518815359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,fp8,63,0.04191466669241587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,fp8,31,0.04399999976158142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,float16,63,0.14411200086275736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,fp8,63,0.042549331982930504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,float16,127,0.14443199833234152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,float16,127,0.14518400033315024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,fp8,127,0.059717332323392235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,1,0.013776000589132309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,fp8,127,0.05932266513506571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,1,0.013765333841244379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,1,0.013679999858140945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,1,0.013882666826248169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,3,0.013754667093356451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,3,0.013744000345468521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,3,0.013967999567588171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,3,0.013637332866589228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,7,0.014314666390419006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,7,0.013973332941532135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,7,0.014208000153303146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,7,0.014170666535695394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,15,0.014655999839305878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,15,0.014592000593741735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,15,0.0145066666106383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,15,0.014752000570297241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,31,0.017898666361967724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,31,0.018789333601792652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,31,0.01806933308641116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,31,0.01802666609485944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,63,0.01809599995613098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,63,0.01802666609485944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,63,0.01842133328318596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,63,0.018197332819302876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,127,0.018325333793958027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,127,0.018453333526849747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,127,0.01829333355029424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,127,0.018383999665578205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,511,0.018453333526849747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,255,0.018298666924238205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,255,0.021754667162895203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,255,0.018245333184798557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,255,0.0216799999276797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,511,0.01828266680240631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,511,0.03156266609827677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,511,0.03213333338499069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,1023,0.018511999398469925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,1023,0.051216001311937966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,1023,0.018629333625237148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,1023,0.05243200063705444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,2047,0.018533332894245785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,2047,0.09000533819198608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,2047,0.018533332894245785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,2047,0.09356799721717834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,4095,0.01851733277241389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,4095,0.16830933094024658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,4095,0.01863466699918111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,4095,0.17450666427612305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,16383,0.018506667266289394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,8191,0.018533332894245785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,8191,0.018581333259741466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,8191,0.3306666612625122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,8191,0.3389973243077596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,16383,0.018543999642133713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,1,0.013776000589132309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,1,0.013733333597580591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,16383,0.7354880174001058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,1,0.013717333475748697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,1,0.01370666672786077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,16383,0.7755733331044515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,3,0.013733333597580591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,3,0.013839999834696451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,3,0.013818666338920593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,3,0.01392000044385592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,7,0.014159999787807465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,7,0.01402666668097178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,7,0.014165333161751429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,7,0.014256000518798828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,15,0.014581333845853806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,15,0.014767999450365702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,15,0.01463466634353002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,15,0.014698666830857595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,31,0.018053332964579265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,31,0.017984000345071156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,31,0.01777600000301997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,31,0.01803733284274737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,63,0.018138666947682697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,63,0.018325333793958027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,63,0.01823466643691063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,63,0.018239999810854595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,127,0.018298666924238205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,127,0.018320000420014065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,127,0.018186666071414948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,127,0.01841066653529803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,255,0.018394666413466137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,255,0.0216799999276797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,255,0.01836266616980235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,255,0.021989333132902782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,511,0.01844800015290578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,511,0.03140799949566523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,511,0.018250666558742523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,511,0.03194133440653483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,1023,0.0184906671444575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,1023,0.051407997806866966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,1023,0.018437333405017853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,1023,0.05239999790986379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,2047,0.018357332795858383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,2047,0.09013332923253377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,2047,0.018415999909241993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,2047,0.09366400043169658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,4095,0.01859733338157336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,4095,0.16924800475438437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,4095,0.018538666268189747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,4095,0.17531200249989828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,8191,0.018181333939234417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,8191,0.018592000007629395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,8191,0.3314613302548726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,8191,0.3405439853668213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,16383,0.01836799954374631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,16383,0.01834133391578992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,1,0.013818666338920593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,1,0.013722666849692663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,1,0.008207999790708223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,1,0.007914666707317034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,16383,0.8600373268127441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,3,0.0138026662170887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,3,0.013807999591032663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,3,0.00785600021481514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,3,0.00810666692753633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,16383,0.7743146419525146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,7,0.014127999544143677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,7,0.014170666535695394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,15,0.007850666840871176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,15,0.008447999755541483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,7,0.008485333373149237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,7,0.008576000109314919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,15,0.014538666854302088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,15,0.014666666587193808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,31,0.01800000046690305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,31,0.01781333362062772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,31,0.008453333129485449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,31,0.008469333251317343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,63,0.01821333294113477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,63,0.018063999712467194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,63,0.008127999802430471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,63,0.0085333331177632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,127,0.018144000321626663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,127,0.01844800015290578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,127,0.009359999870260557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,127,0.009413333609700203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,255,0.013376000026861826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,255,0.01312000056107839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,255,0.009546666716535887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,255,0.01033599985142549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,511,0.013674666484196981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,511,0.013834666460752487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,511,0.009461333354314169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,511,0.010629333555698395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,1023,0.018906666586796444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,1023,0.019941333681344986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,1023,0.010159999753038088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,1023,0.013072000195582708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,2047,0.021770666042963665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,2047,0.02740799884001414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,2047,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,2047,0.017605333278576534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,4095,0.02698666602373123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,4095,0.03755199909210205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,8191,0.05460800230503082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,8191,0.01062400018175443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,8191,0.03619199991226196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,4095,0.010357333347201347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,4095,0.026474667092164356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,8191,0.03177600105603536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,16383,0.0393653338154157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,1,0.010309333602587381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,16383,0.08235733211040497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,16383,0.0106133334338665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,1,0.010437333335479101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,16383,0.05888533095518748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,1,0.006698666761318843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,1,0.006677333265542984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,3,0.010357333347201347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,3,0.01044800008336703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,3,0.006741333131988843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,3,0.006853333363930385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,7,0.010368000095089277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,7,0.010496000448862711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,7,0.006618666773041089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,7,0.006714666883150737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,15,0.010239999741315842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,15,0.01044800008336703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,15,0.006618666773041089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,63,0.0102613332370917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,15,0.006858666737874349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,31,0.010298666854699453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,31,0.010570666442314783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,31,0.00684799998998642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,31,0.006757333253820737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,63,0.010362666721145311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,63,0.006693333387374878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,63,0.006805333619316419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,127,0.010250666489203772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,127,0.010314666976531347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,127,0.007578666632374127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,127,0.007381333038210869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,255,0.012293333808581034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,255,0.01246400053302447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,255,0.008069333309928576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,255,0.008645333349704742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,511,0.01700266698996226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,511,0.01764800027012825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,511,0.008746666833758354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,511,0.009839999799927076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,1023,0.02221333235502243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,1023,0.025242666403452556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,1023,0.00890666681031386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,1023,0.01302933320403099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,4095,0.05273066461086273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,2047,0.02749866743882497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,2047,0.03606933355331421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,2047,0.009205333267649015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,2047,0.02060266708334287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,4095,0.034304000437259674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,4095,0.009413333609700203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,4095,0.026394667724768322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,8191,0.03444266567627589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,8191,0.06810666620731354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,8191,0.009152000149091085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,8191,0.036570665736993156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,16383,0.03438399980465571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,16383,0.09063466389973958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,16383,0.009152000149091085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,16383,0.05834133426348368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,float16,1,1.1687040328979492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,float16,1,1.1742986838022869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,fp8,1,1.1845760345458984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,fp8,1,1.1826186974843342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,float16,3,1.1819840272267659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,float16,3,1.1708160241444905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,fp8,3,1.188490629196167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,fp8,3,1.1757919788360596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,float16,7,1.2327306270599365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,float16,7,1.2411626974741619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,fp8,7,1.2457066377003987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,fp8,7,1.2454826831817627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,float16,15,1.2969653606414795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,float16,15,1.2865066528320312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,fp8,15,1.301754633585612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,fp8,15,1.289690653483073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,float16,31,1.6521387100219727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,float16,31,1.6593866348266602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,fp8,31,1.6686879793802898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,fp8,31,1.6668160756429036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,float16,63,1.6805280049641926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,float16,63,1.6701119740804036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,fp8,63,1.6847039858500164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,fp8,63,1.6713600158691406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,float16,1,1.1908746560414631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,float16,1,1.1989013353983562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,fp8,1,1.2053653399149578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,fp8,1,1.2021493117014568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,float16,3,1.2055946985880535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,float16,3,1.1954560279846191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,fp8,3,1.2093706925710042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,fp8,3,1.1977333227793376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,float16,7,1.2402986685434978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,float16,7,1.2497333685557048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,fp8,7,1.2549333572387695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,fp8,7,1.2539733250935872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,float16,15,1.3146080176035564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,float16,15,1.3066826661427815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,fp8,15,1.3159626324971516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,fp8,15,1.3059893449147542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,float16,31,1.6682880719502766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,float16,31,1.6802666982014973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,fp8,31,1.6856800715128581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,fp8,31,1.684175968170166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,float16,63,1.6991039911905925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,float16,63,1.6880106925964355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,fp8,63,1.7033813794453938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,float16,1,0.18595200777053833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,fp8,1,0.08985599875450134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,float16,1,0.18624534209569296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,fp8,1,0.09012266993522644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,fp8,63,1.691088040669759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,float16,3,0.18677333990732828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,fp8,3,0.0895146628220876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,float16,3,0.1852746605873108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,fp8,3,0.08987200260162354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,float16,7,0.188426673412323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,float16,7,0.1867199937502543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,fp8,7,0.09002133210500081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,fp8,7,0.08898133039474487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,float16,15,0.18761066595713297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,float16,15,0.1880799929300944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,fp8,15,0.0892693301041921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,fp8,15,0.08975467085838318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,float16,31,0.18836800257364908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,float16,31,0.18752533197402954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,fp8,31,0.09001066287358601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,float16,63,0.1857973337173462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,fp8,31,0.08913600444793701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,float16,63,0.1868106722831726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,fp8,63,0.08872000376383464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,fp8,63,0.08873066306114197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,float16,1,0.28116800387700397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,fp8,1,0.09246933460235596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,float16,1,0.28175467252731323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,fp8,1,0.09240000446637471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,float16,3,0.2825653354326884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,fp8,3,0.09248532851537068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,float16,3,0.2812853256861369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,fp8,3,0.09219732880592346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,float16,7,0.2818400065104167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,float16,7,0.2826506694157918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,fp8,7,0.09214400251706441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,fp8,7,0.09261866410573323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,float16,15,0.28464533885320026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,fp8,15,0.09222400188446045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,float16,15,0.2835413416226705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,fp8,15,0.09233066439628601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,float16,31,0.28650132815043133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,float16,31,0.2844480077425639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,fp8,31,0.09193600217501323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,fp8,31,0.09160533547401428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,float16,63,0.28060799837112427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,fp8,63,0.09113066395123799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,float16,63,0.28117332855860394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,fp8,63,0.0916319986184438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,1,64,128,1,float16,float16,1,2.3905653953552246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,float16,1,2.3915680249532065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,1,64,128,1,float16,fp8,1,2.408522605895996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,fp8,1,2.40993595123291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,1,64,128,1,float16,float16,3,2.395696004231771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,float16,3,2.394165356953939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,1,64,128,1,float16,fp8,3,2.4129652976989746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,fp8,3,2.412447929382324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,1,64,128,1,float16,float16,7,2.496917406717936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,1,64,128,1,float16,fp8,7,2.506341298421224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,fp8,7,2.5066986083984375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,1,64,128,1,float16,float16,15,2.5963093439737954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,float16,15,2.5961546897888184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,1,64,128,1,float16,fp8,15,2.6139039993286133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,fp8,15,2.6187307039896646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,1,64,128,1,float16,float16,31,3.312800089518229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,float16,31,3.313237190246582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,2,64,128,1,float16,float16,1,2.4369972546895347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,1,64,128,1,float16,fp8,31,3.3434985478719077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,float16,1,2.4379626909891763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,fp8,31,3.343658765157064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,2,64,128,1,float16,fp8,1,2.460949261983236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,fp8,1,2.45141871770223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,2,64,128,1,float16,float16,3,2.441333293914795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,float16,3,2.4415946006774902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,2,64,128,1,float16,fp8,3,2.456005255381266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,fp8,3,2.456069310506185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,2,64,128,1,float16,float16,7,2.500293254852295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,float16,7,2.517162640889486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,float16,7,2.4979680379231772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,2,64,128,1,float16,fp8,7,2.516757329305013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,fp8,7,2.5167360305786133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,2,64,128,1,float16,float16,15,2.6280105908711753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,float16,15,2.625760078430176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,2,64,128,1,float16,fp8,15,2.6385439236958823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,fp8,15,2.638773282368978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,4,64,128,1,float16,float16,1,0.36765865484873456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,float16,1,0.36716266473134357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,4,64,128,1,float16,fp8,1,0.16843199729919434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,fp8,1,0.16833066940307617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,2,64,128,1,float16,float16,31,3.361061414082845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,float16,31,3.33679993947347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,fp8,31,3.364159901936849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,2,64,128,1,float16,fp8,31,3.3641173044840493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,4,64,128,1,float16,float16,3,0.3677866856257121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,4,64,128,1,float16,fp8,3,0.1681333382924398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,float16,3,0.36661334832509357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,fp8,3,0.16793600718180338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,4,64,128,1,float16,float16,7,0.3693973223368327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,float16,7,0.36692798137664795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,4,64,128,1,float16,fp8,7,0.1676373283068339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,fp8,7,0.16854933897654215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,4,64,128,1,float16,float16,15,0.36663464705149335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,4,64,128,1,float16,fp8,15,0.16755733887354532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,float16,15,0.36802132924397785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,fp8,15,0.16817599534988403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,4,64,128,1,float16,float16,31,0.36830933888753253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,float16,31,0.36580268541971844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,4,64,128,1,float16,fp8,31,0.1670080025990804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,fp8,31,0.16834133863449097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,8,64,128,1,float16,fp8,1,0.17679466803868613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,fp8,1,0.17642132441202799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,8,64,128,1,float16,float16,1,0.562602678934733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,float16,1,0.5610933303833008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,8,64,128,1,float16,float16,3,0.5614346663157145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,8,64,128,1,float16,fp8,3,0.17624000708262125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,fp8,3,0.17693867286046347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,float16,3,0.5618506669998169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,8,64,128,1,float16,fp8,7,0.17575466632843018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,fp8,7,0.17601599295934042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,float16,7,0.5634826819101969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,8,64,128,1,float16,float16,7,0.5657920042673746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,8,64,128,1,float16,float16,15,0.5620533227920532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,8,64,128,1,float16,fp8,15,0.1769919991493225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,float16,15,0.561519980430603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,1,0.023445333043734234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,1,0.02327999969323476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,fp8,15,0.1758400003115336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,1,0.023621333142121632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,1,0.023578666150569916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,3,0.02346133440732956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,8,64,128,1,float16,fp8,31,0.1763040026028951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,3,0.023381332556406658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,8,64,128,1,float16,float16,31,0.5614879926045736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,3,0.02351466566324234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,float16,31,0.5634933312733968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,3,0.02366400013367335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,7,0.024090667565663654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,7,0.024192000428835552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,fp8,31,0.17554134130477905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,7,0.024480000138282776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,7,0.024330665667851765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,15,0.024933333198229473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,15,0.025205334027608235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,15,0.024901332954565685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,15,0.02526933451493581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,31,0.030720000465710957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,31,0.03105599929889043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,31,0.030991998811562855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,31,0.030981334547201794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,63,0.03159466634194056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,63,0.031152000029881794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,63,0.03151999910672506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,63,0.031685332457224526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,127,0.03173333406448364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,127,0.03178133318821589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,127,0.03176533430814743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,127,0.031770666440327965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,511,0.05671466886997223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,255,0.03196800003449122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,255,0.03807999938726425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,255,0.031898667414983116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,255,0.03794133414824804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,511,0.03214933226505915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,511,0.031898667414983116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,511,0.05775466561317444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,2047,0.1710346738497416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,1023,0.031845333675543465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,1023,0.09520000219345093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,1023,0.031871999303499855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,4095,0.03201599915822347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,2047,0.17483733097712198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,1023,0.0971999963124593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,2047,0.032074667513370514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,2047,0.031957333286603294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,4095,0.032127998769283295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,4095,0.3225173354148865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,4095,0.33002666632334393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,8191,0.03189333279927572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,1,0.023647998770078022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,8191,0.03192000091075897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,1,0.023552000522613525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,8191,0.6343573331832886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,1,0.02372266600529353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,1,0.02365333338578542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,3,0.023370665808518726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,8191,0.6437706549962362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,3,0.02362666775782903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,7,0.024421334266662598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,3,0.023567999402681988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,3,0.02369066576162974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,7,0.024351999163627625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,7,0.024277334411938984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,7,0.02458133300145467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,15,0.025125332176685333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,15,0.02534399926662445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,15,0.02510400116443634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,15,0.02513599892457326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,31,0.031248000760873158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,63,0.03150933235883713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,31,0.031167998909950256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,127,0.03172266731659571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,31,0.031157332162062328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,31,0.03138133386770884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,63,0.031504000226656594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,63,0.031898667414983116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,63,0.03156800071398417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,127,0.03200533241033554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,127,0.031845333675543465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,127,0.03182400017976761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,255,0.03178666780392329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,255,0.03793599953254064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,255,0.03195200115442276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,255,0.03822933385769526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,511,0.032032000521818794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,511,0.05712000032265981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,511,0.03198933353026708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,511,0.057461331288019814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,1023,0.03196800003449122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,1023,0.0950879951318105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,1023,0.032144000132878624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,1023,0.0979306697845459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,2047,0.03209066639343897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,2047,0.031957333286603294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,2047,0.1713493267695109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,2047,0.17713600397109985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,4095,0.032058666149775185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,4095,0.03219199925661087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,4095,0.3230666716893514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,1,0.011701333026091257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,8191,0.032069332897663116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,4095,0.33524266878763836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,8191,0.03193599979082743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,8191,0.6474773486455282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,1,0.011690666278203329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,1,0.008272000278035799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,3,0.008634666601816813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,1,0.008272000278035799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,8191,0.6447786490122477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,3,0.011551999797423681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,3,0.011616000284751257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,15,0.011472000430027643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,3,0.008143999924262365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,7,0.011829332758982977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,7,0.011744000017642975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,7,0.008442666381597519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,7,0.008181333541870117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,15,0.011557333171367645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,15,0.00784533346692721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,31,0.008165333420038223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,15,0.00816000004609426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,31,0.011866666376590729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,31,0.011744000017642975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,31,0.008576000109314919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,63,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,63,0.011365332951148352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,63,0.008341333518425623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,63,0.008538666491707167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,127,0.011685332904259363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,127,0.011498666057984034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,255,0.009530666594703993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,127,0.008885333314538002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,127,0.009397333487868309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,255,0.013760000467300415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,255,0.013850666582584381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,255,0.010559999694426855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,511,0.019365333020687103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,511,0.021690666675567627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,1023,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,511,0.010165333126982054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,511,0.011829332758982977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,1023,0.02532266577084859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,1023,0.029045333464940388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,1023,0.021216000119845074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,2047,0.03164266546567281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,2047,0.04188266893227895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,2047,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,2047,0.024458666642506916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,4095,0.0396373321612676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,4095,0.0633546660343806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,4095,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,4095,0.03888533264398575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,8191,0.039477333426475525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,8191,0.08076799909273784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,8191,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,1,0.010581333190202713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,8191,0.05736533304055532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,1,0.010682666053374609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,1,0.00679466687142849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,1,0.006645333642760913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,3,0.010543999572594961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,3,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,3,0.0069226666043202085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,3,0.006725333631038666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,7,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,7,0.010703999549150467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,7,0.006768000001708667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,7,0.006704000135262807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,15,0.010522666076819101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,15,0.010650667051474253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,15,0.006911999856432279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,15,0.0069759997228781385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,63,0.010640000303586325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,31,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,31,0.011823999385039011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,31,0.006735999758044879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,31,0.006688000013430913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,63,0.010490667074918747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,63,0.006874666859706243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,63,0.006906666482488315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,127,0.010512000570694605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,127,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,127,0.007360000163316727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,127,0.007386666412154834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,255,0.01834133391578992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,255,0.018474667022625606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,255,0.00878399983048439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,255,0.009637333452701569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,511,0.023061332603295643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,511,0.025573333104451496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,511,0.009610666582981745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,511,0.011653333902359009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,1023,0.03334933271010717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,1023,0.040250666439533234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,2047,0.009722666814923286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,1023,0.009872000043590864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,1023,0.02019199977318446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,2047,0.035429333647092186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,4095,0.06683733562628429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,2047,0.05263466636339823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,2047,0.02420266717672348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,4095,0.03566399961709976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,4095,0.00979200005531311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,4095,0.03483733286460241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,8191,0.00979200005531311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,8191,0.035616000493367515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,8191,0.05563733478387197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,8191,0.0890826682249705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,1,0.04154666761557261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,1,0.04273599882920583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,3,0.044010668992996216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,3,0.04426133135954539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,7,0.044581333796183266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,31,0.056741332014401756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,7,0.045184001326560974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,15,0.05611733098824819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,15,0.05606399973233541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,31,0.056874667604764305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,63,0.05702400207519531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,63,0.05657599866390228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,127,0.06934399902820587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,127,0.0682826687892278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,255,0.10634666681289673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,255,0.10598933696746826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,511,0.18486400445302328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,511,0.18303465843200684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,1023,0.3351200024286906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,1023,0.3370826641718547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,2047,0.6386719942092896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,1,0.04182399809360504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,1,0.04253333310286204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,2047,0.6392639875411987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,3,0.044266665975252785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,3,0.044293334086736046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,7,0.0452159990866979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,7,0.04568533102671305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,4095,1.2512799898783367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,4095,1.2562399705251057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,15,0.05707733333110809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,15,0.05729599793752035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,31,0.05719466507434845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,31,0.05717866619427999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,63,0.05746666590372721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,63,0.05737066765626272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,127,0.07041066884994507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,127,0.06958933174610138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,255,0.10929600397745769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,255,0.1076586643854777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,511,0.18514132499694824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,511,0.18436266978581747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,1023,0.3369813362757365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,1023,0.33904000123341876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,2047,0.6451199849446615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,2047,0.6417973438898722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,4095,1.2569440205891926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,4095,1.2666880289713542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,1,0.00973866693675518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,15,0.009626666704813639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,31,0.009488000224033991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,3,0.009488000224033991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,63,0.009488000224033991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,7,0.009722666814923286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,127,0.010437333335479101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,1,0.01322666679819425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,3,0.013327999661366144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,511,0.035418666899204254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,7,0.013440000514189402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,1023,0.05909866591294607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,15,0.013151999562978745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,2047,0.08349866668383281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,31,0.013359999905029932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,63,0.01312000056107839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,4095,0.10957866907119751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,4095,0.08297066887219746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,127,0.012863999853531519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,255,0.02625600000222524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,255,0.013818666338920593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,511,0.0249439999461174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,1023,0.04400533437728882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,2047,0.06438933312892914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,3,0.008309333274761835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,1,0.008090666805704435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,31,0.00795199970404307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,7,0.008192000289758047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,63,0.007962666451931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,15,0.007930666829148928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,127,0.008896000062425932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,1,0.0163680004576842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,3,0.01605333387851715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,31,0.01635733370979627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,511,0.04965866605440775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,63,0.015872000406185787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,511,0.026741333305835724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,7,0.016506666938463848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,1023,0.06448000172773997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,127,0.016602666427691776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,1023,0.03730133424202601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,2047,0.08653866251309712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,2047,0.05221333106358846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,4095,0.12967999776204428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,1,0.009141333401203156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,4095,0.07710933188597362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,1,0.00926399976015091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,3,0.009232000137368837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,3,0.009109333157539368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,7,0.0100426667680343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,7,0.00914666677514712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,31,0.010058666889866194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,15,0.009359999870260557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,15,0.00949866697192192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,31,0.009989333028594652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,63,0.011488000551859537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,63,0.011616000284751257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,15,0.016271999726692837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,127,0.011663999408483505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,127,0.011477333803971609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,255,0.011770666887362799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,255,0.011642667154471079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,255,0.031370667119820915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,511,0.026005332668622334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,511,0.025994665920734406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,1023,0.030095999439557392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,1023,0.030159999926884968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,2047,0.04030400017897288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,2047,0.04009066770474116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,4095,0.060778667529424034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,4095,0.06071466704209646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,8191,0.1014400025208791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,8191,0.10056533416112264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,1,0.00895999992887179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,16383,0.18225600322087607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,1,0.009183999771873156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,16383,0.177946666876475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,15,0.009328000247478485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,3,0.009114666531483332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,3,0.009029333169261614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,7,0.008943999807039896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,7,0.009029333169261614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,15,0.009392000113924345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,31,0.010015999898314476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,31,0.00979200005531311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,63,0.011610666910807291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,63,0.01145600030819575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,511,0.02589333305756251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,127,0.011477333803971609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,127,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,255,0.011813333878914515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,255,0.01830400029818217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,255,0.011701333026091257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,511,0.02604266752799352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,1023,0.030074665943781536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,1023,0.03009066730737686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,8191,0.1002293328444163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,2047,0.04048533240954081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,2047,0.040074666341145836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,4095,0.06043733159701029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,4095,0.05996266504128774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,8191,0.1015786627928416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,1,0.009242666885256767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,16383,0.18234666188557944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,16383,0.1779680053393046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,1,0.008901333436369896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,3,0.008954666554927826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,3,0.009061333412925402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,7,0.008863999818762144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,7,0.00916800027092298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,15,0.009408000235756239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,15,0.009077333534757296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,31,0.009824000298976898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,31,0.009535999968647957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,63,0.011402666568756104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,63,0.008997333546479544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,127,0.011413333316644033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,1023,0.02978666623433431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,127,0.010474666953086853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,2047,0.015487999965747198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,255,0.011861333002646765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,255,0.011717333147923151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,511,0.025888000925381977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,511,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,8191,0.021754667162895203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,1023,0.014378666877746582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,2047,0.017781333376963932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,4095,0.017818666994571686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,4095,0.01855466639002164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,8191,0.023205332458019257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,16383,0.02515200028816859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,16383,0.02607999990383784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,1,0.008858666444818178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,3,0.008896000062425932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,15,0.00690133310854435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,3,0.006895999734600385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,31,0.00985599992175897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,7,0.00898133342464765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,7,0.007130666946371396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,15,0.009178666397929192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,31,0.006906666482488315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,63,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,63,0.007296000296870868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,127,0.011503999431928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,127,0.00766933336853981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,255,0.011621333658695221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,255,0.009354666496316591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,511,0.02586666742960612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,511,0.01022933361430963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,1023,0.013861333330472311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,1023,0.010629333555698395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,2047,0.015168000012636185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,2047,0.01163200040658315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,8191,0.0183146670460701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,16383,0.03127466638882955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,4095,0.01617066686352094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,4095,0.01358933374285698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,8191,0.02587199956178665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,16383,0.021712000171343487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,1,0.00855466661353906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,1,0.008618666479984919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,3,0.008559999987483025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,3,0.008725333337982496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,7,0.00871999996403853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,7,0.008458666503429413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,15,0.009482666850090027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,15,0.00955200009047985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,31,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,31,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,63,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,63,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,127,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,127,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,255,0.012773333738247553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,255,0.012810666114091873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,511,0.017759999881188076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,511,0.018101333330074947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,1023,0.02777066578467687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,1023,0.028234665592511494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,1,0.007082666580875714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,8191,0.16633599996566772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,2047,0.04760533571243286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,2047,0.047983999053637184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,4095,0.08701866865158081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,4095,0.08777067065238953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,8191,0.16799465815226236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,1,0.008602666358153025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,16383,0.3256266713142395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,1,0.008725333337982496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,3,0.008581333483258883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,16383,0.32416532437006634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,3,0.008874666566650072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,7,0.008645333349704742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,7,0.008592000231146812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,15,0.009423999736706415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,15,0.009423999736706415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,31,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,31,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,63,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,63,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,127,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,127,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,255,0.013045333325862885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,255,0.012975999464591345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,511,0.017802666872739792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,511,0.017887999614079792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,4095,0.08854400118192036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,1023,0.027786667148272198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,1023,0.028277332584063213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,2047,0.04747733473777771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,2047,0.04827733337879181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,1,0.008469333251317343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,1,0.009098666409651438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,4095,0.08819733063379924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,8191,0.16638400157292685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,8191,0.1667626698811849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,16383,0.333024005095164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,16383,0.3240373333295186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,31,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,3,0.008517333616813024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,3,0.00973866693675518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,63,0.00891733355820179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,7,0.008650666723648706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,127,0.01126933346192042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,7,0.009701333319147428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,15,0.009408000235756239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,15,0.009573333586255709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,31,0.00914666677514712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,63,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,127,0.010186666622757912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,1023,0.013834666460752487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,255,0.012885333349307379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,255,0.012618667135636011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,4095,0.020554666717847187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,511,0.018005333840847015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,4095,0.017973333597183228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,511,0.012885333349307379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,8191,0.024842667082945507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,1023,0.016149333367745083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,2047,0.01851733277241389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,2047,0.015626666446526844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,16383,0.032416000962257385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,8191,0.03186666717131933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,16383,0.03885866701602936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,1,0.008549333239595095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,1,0.007093333328763644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,3,0.008762666955590248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,3,0.007237333183487256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,7,0.009839999799927076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,7,0.007146666447321574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,15,0.009296000003814697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,15,0.0074346667776505155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,31,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,31,0.007226666435599327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,63,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,127,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,255,0.013013333082199097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,255,0.00972800018886725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,511,0.013541333377361298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,511,0.009701333319147428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,1023,0.013728000223636627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,1023,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,2047,0.0206133338312308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,2047,0.013386666774749756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,4095,0.0240639994541804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,4095,0.01545599972208341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,8191,0.03522133330504099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,8191,0.02170666555563609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,16383,0.05442666510740916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,1,0.07764266431331635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,1,0.08025600016117096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,16383,0.036362667878468834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,3,0.08099199831485748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,3,0.08321066697438557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,7,0.08434133728345235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,7,0.08627733588218689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,15,0.10822400450706482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,31,0.10867733756701152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,15,0.10688533385594685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,31,0.10687466462453206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,63,0.10903466741243999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,63,0.10821866989135742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,127,0.13249599933624268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,127,0.1309813360373179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,255,0.2066133419672648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,255,0.20578666528066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,511,0.358458677927653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,511,0.357258677482605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,1,0.07897066573301952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,1023,0.6521653334299723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,1023,0.6561866601308187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,1,0.08094933132330577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,3,0.08125866452852885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,3,0.08362133304278056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,7,0.08719999591509502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,7,0.08500799536705017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,2047,1.2524800300598145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,15,0.10832533240318298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,15,0.10884799559911092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,31,0.1095413366953532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,2047,1.2541600068410237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,31,0.10801066954930623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,63,0.10917333761850993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,63,0.10869866609573364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,127,0.13451199730237326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,255,0.20757333437601724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,127,0.13384000460306802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,255,0.20932267109553018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,511,0.3567733367284139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,511,0.35682666301727295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,1,0.020021333048741024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,1,0.01044800008336703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,1023,0.6650293270746866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,1023,0.65229864915212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,3,0.020154666155576706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,3,0.010255999863147736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,7,0.020560000091791153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,7,0.010181333248813948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,15,0.020213333268960316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,31,0.020629333953062694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,63,0.007514666765928268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,2047,1.2661279837290447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,63,0.020245333512624104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,15,0.010368000095089277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,2047,1.2640480200449626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,31,0.010197333370645842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,63,0.010421333213647207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,127,0.020389333367347717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,127,0.011674666156371435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,255,0.039034667114416756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,1023,0.07944533228874207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,127,0.008037333066264788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,255,0.02348266790310542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,511,0.06147199869155884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,511,0.04452799757321676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,1023,0.0651146670182546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,1,0.026741333305835724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,2047,0.11076800028483073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,2047,0.08224000036716461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,1,0.012768000364303589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,3,0.026709333062171936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,3,0.012730666746695837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,7,0.026677332818508148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,7,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,15,0.02703999976317088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,15,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,31,0.026378666361172993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,31,0.012821332861979803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,63,0.02625600000222524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,63,0.012725333372751871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,127,0.026719999810059864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,127,0.01544533297419548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,255,0.053120002150535583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,255,0.026821332673231762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,511,0.06791999936103821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,511,0.03698666642109553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,1023,0.08709866801897685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,1023,0.05258666475613912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,2047,0.13477333386739096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,2047,0.07806399961312611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,7,0.01392000044385592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,1,0.013429333766301474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,15,0.015098666151364645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,1,0.013562666873137156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,3,0.013717333475748697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,3,0.013898666948080063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,7,0.013904000322024027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,15,0.014965333044528961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,31,0.01806933308641116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,31,0.018415999909241993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,63,0.0183999997874101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,63,0.01828266680240631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,127,0.018453333526849747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,127,0.01869333287080129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,1023,0.050944000482559204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,255,0.021882665654023487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,255,0.021957332889238994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,511,0.03190933416287104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,511,0.031845333675543465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,4095,0.16833066940307617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,1023,0.05142933130264282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,2047,0.09107733766237895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,8191,0.32150934139887494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,2047,0.08980266253153484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,4095,0.1670560042063395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,1,0.013429333766301474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,8191,0.32519465684890747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,1,0.013760000467300415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,3,0.013621332744757334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,3,0.013999999811251959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,16383,0.6311359802881876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,7,0.013760000467300415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,7,0.013973332941532135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,63,0.01848000039656957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,16383,0.6391893227895101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,15,0.015040000279744467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,15,0.015157333264748255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,31,0.017994667092959087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,31,0.018298666924238205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,63,0.018272000054518383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,127,0.01854933301607768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,127,0.018511999398469925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,255,0.021930667261282604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,255,0.022240000466505688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,511,0.03215999901294708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,511,0.031701333820819855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,1023,0.051226665576299034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,1023,0.05136000116666158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,2047,0.09010666608810425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,2047,0.09033067027727763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,4095,0.16942399740219116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,4095,0.1691946585973104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,8191,0.32923734188079834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,1,0.013616000612576803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,8191,0.3223946690559387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,1,0.009183999771873156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,3,0.013765333841244379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,3,0.009743999689817429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,7,0.014074667046467463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,16383,0.6481599807739258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,16383,0.6311253309249878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,7,0.009690666571259499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,15,0.014767999450365702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,15,0.009125333279371262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,31,0.018218666315078735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,31,0.00985599992175897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,63,0.018629333625237148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,63,0.009685333197315535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,127,0.01833600054184596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,127,0.010181333248813948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,255,0.021776000658671062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,255,0.011999999483426413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,511,0.015360000232855478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,511,0.012837332983811697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,1023,0.01632000009218852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,1023,0.013904000322024027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,2047,0.02516799916823705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,2047,0.017418666432301205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,4095,0.02942399928967158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,4095,0.020762667059898376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,8191,0.04660800099372864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,8191,0.0365280012289683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,16383,0.06727999945481618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,16383,0.05034666756788889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,1,0.013637332866589228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,1,0.0069866664707660675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,3,0.013786666095256805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,3,0.007397333160042763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,7,0.01413333291808764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,7,0.007365333537260692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,15,0.014805333067973455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,15,0.00730666642387708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,31,0.018218666315078735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,31,0.007189333438873291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,63,0.01865600049495697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,63,0.007525333513816197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,127,0.01854933301607768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,127,0.007941333577036858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,255,0.013199999928474426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,255,0.009429333110650381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,2047,0.02790933350721995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,511,0.013728000223636627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,511,0.010010666524370512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,1023,0.020037333170572918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,1023,0.012154666086037954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,2047,0.015306666493415833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,4095,0.041519999504089355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,4095,0.024373332659403484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,8191,0.059994667768478394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,1,0.1497599979241689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,1,0.15357866883277893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,8191,0.031845333675543465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,16383,0.09133866429328918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,16383,0.05671999851862589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,3,0.1564906636873881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,3,0.160671999057134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,7,0.16351466377576193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,7,0.166293332974116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,15,0.2116533319155375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,15,0.21107733249664307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,31,0.21030400196711221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,31,0.20892266432444254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,63,0.2127093275388082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,63,0.21039466063181558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,127,0.2566240032513936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,127,0.2587520082791646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,255,0.40720534324645996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,255,0.40493865807851154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,511,0.7036373615264893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,1,0.15103999773661295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,511,0.6997439861297607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,1,0.15627732872962952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,3,0.15826132893562317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,3,0.16025599837303162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,7,0.16461333632469177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,1023,1.3036853472391765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,7,0.1672853430112203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,1023,1.285370667775472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,15,0.2111626664797465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,15,0.21220799287160239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,31,0.21302932500839233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,31,0.21106666326522827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,63,0.2130026618639628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,63,0.21332800388336182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,127,0.2598186731338501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,127,0.25971200068791706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,255,0.40746132532755536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,255,0.4102613528569539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,511,0.7150826454162598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,511,0.7021493117014567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,1,0.03465066601832708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,1,0.017535999417304993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,3,0.034314667185147606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,7,0.017360000560681026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,1023,1.2989813486735027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,3,0.017583999782800674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,31,0.017386666188637417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,7,0.03508266558249792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,1023,1.3056639830271404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,15,0.03455466777086258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,127,0.020021333048741024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,15,0.017445333302021027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,31,0.03451200077931086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,63,0.03440533330043157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,63,0.01735466718673706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,127,0.03419200082619985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,255,0.06774400174617767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,1,0.046906664967536926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,255,0.047728002071380615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,511,0.08477866649627686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,3,0.018581333259741466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,511,0.0672266681989034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,1023,0.11079466342926025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,1023,0.08454933762550354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,1,0.018474667022625606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,15,0.018357332795858383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,3,0.04722133278846741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,7,0.04647466540336609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,7,0.01863466699918111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,15,0.046896000703175865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,31,0.04696533580621084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,31,0.018346666047970455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,63,0.04655999938646952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,63,0.018570666511853535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,127,0.048826664686203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,127,0.023904000719388325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,255,0.05878399809201559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,255,0.034314667185147606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,511,0.07861333092053731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,511,0.04874666531880697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,1023,0.12572800119717917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,1023,0.07234133283297221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,1,0.2943519949913025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,3,0.30745599667231244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,fp8,1,0.30245866378148395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,fp8,3,0.31330666939417523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,7,0.3214079936345418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,fp8,7,0.3293066620826721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,15,0.4145600001017253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,fp8,15,0.4160319964090983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,31,0.41484800974527997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,fp8,31,0.41619733969370526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,63,0.4171626567840576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,fp8,63,0.41522133350372314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,127,0.5110559860865275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,fp8,127,0.5084319909413656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,1,0.29811733961105347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,fp8,1,0.30481066306432086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,3,0.30832000573476154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,255,0.813045342763265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,fp8,255,0.8019733428955078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,fp8,3,0.31677865982055664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,7,0.32330665985743207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,fp8,7,0.33139199018478394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,15,0.4172319968541463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,fp8,15,0.41570667425791424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,31,0.42084264755249023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,fp8,31,0.41549332936604816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,63,0.42275198300679523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,fp8,63,0.41813866297403973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,127,0.5263733466466268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,1,0.06303466856479645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,fp8,127,0.5141226847966512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,fp8,1,0.03242666771014532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,3,0.06244266529877981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,255,0.8180960019429525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,fp8,3,0.032698666055997215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,7,0.06301333506902058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,fp8,255,0.8223840395609537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,fp8,7,0.03270933280388514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,31,0.06369066735108693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,15,0.06410666803518932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,fp8,15,0.03213333338499069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,fp8,31,0.03295466552178065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,63,0.06437866886456807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,127,0.06515733400980632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,fp8,63,0.03350933392842611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,fp8,127,0.04242666562398275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,255,0.07683733105659485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,fp8,3,0.035242666800816856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,fp8,255,0.06161599854628245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,1,0.08649599552154541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,fp8,7,0.035631999373435974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,fp8,1,0.036544000109036766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,15,0.08746666709582011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,3,0.08640533685684204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,7,0.0869599978129069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,fp8,15,0.035088000198205314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,31,0.0867733359336853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,fp8,31,0.036805334190527596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,63,0.08739200234413147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,fp8,63,0.03598399957021078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,127,0.08969066540400188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,fp8,127,0.055071999629338585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,255,0.10531733433405559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,fp8,255,0.06474133332570393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,1,0.5869333346684774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,fp8,1,0.605189323425293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,3,0.6136053403218588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,fp8,3,0.6254186630249023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,7,0.6512746810913086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,fp8,7,0.6559733152389526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,15,0.8341973622639974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,fp8,15,0.8236479759216309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,31,0.8330612977345785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,fp8,31,0.831765333811442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,63,0.8426346778869629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,fp8,63,0.8356533050537109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,1,0.6046773195266724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,127,1.0318346818288167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,fp8,127,1.02347199122111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,fp8,1,0.6109439929326376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,3,0.6199359893798828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,fp8,3,0.6346400181452433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,7,0.6542453368504842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,fp8,7,0.6646506786346436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,15,0.8464106718699137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,fp8,15,0.8386452992757162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,31,0.8476266860961914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,fp8,31,0.8374293645222982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,63,0.8613333702087402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,fp8,63,0.8499253590901693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,1,0.12303466598192851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,fp8,1,0.09101333220799764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,127,1.0432746410369873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,3,0.12132267157236735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,fp8,3,0.09196799993515015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,fp8,7,0.09215466181437175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,fp8,127,1.0495839913686116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,7,0.12230400244394939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,15,0.12184000015258789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,fp8,15,0.09185066819190979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,31,0.1221386690934499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,fp8,31,0.09199466307957967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,63,0.12272533774375916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,fp8,127,0.10132799545923869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,fp8,63,0.09186133742332458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,127,0.12246400117874146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,fp8,1,0.0828959991534551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,3,0.16762133439381918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,1,0.16716265678405762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,fp8,3,0.08341333270072937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,7,0.1672053337097168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,fp8,7,0.0835040012995402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,15,0.1673120061556498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,31,0.16753600041071573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,fp8,15,0.08302400012811025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,fp8,31,0.08343999584515889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,1,0.013557333499193192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,63,0.16832000017166138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,1,0.013744000345468521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,fp8,63,0.0823520024617513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,3,0.013882666826248169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,3,0.014282666146755219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,7,0.014453332871198654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,7,0.014389333625634512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,127,0.16988267501195273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,63,0.017877332866191864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,fp8,127,0.09945600231488545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,15,0.01775466650724411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,15,0.017952000101407368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,31,0.017978666971127193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,31,0.017887999614079792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,63,0.018031999468803406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,127,0.02120000123977661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,127,0.021210665504137676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,255,0.0313226655125618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,255,0.031343999008337654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,511,0.051669334371884666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,511,0.05206400156021118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,1023,0.09123733639717102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,1023,0.0936959981918335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,2047,0.16934933265050253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,2047,0.17328532536824545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,4095,0.3263946572939555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,4095,0.33393601576487225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,1,0.014453332871198654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,1,0.013728000223636627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,8191,0.6559253136316935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,8191,0.6596373319625854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,3,0.013850666582584381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,15,0.017909333109855652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,3,0.014117332796255747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,7,0.014352000008026758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,7,0.014544000228246054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,16383,1.6020480791727703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,16383,1.6244319279988606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,15,0.017845333864291508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,31,0.01791999985774358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,31,0.01793066660563151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,63,0.017887999614079792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,63,0.017840000490347546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,127,0.021173333128293354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,127,0.021488000949223835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,255,0.031557333966096245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,255,0.03139200061559677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,511,0.051114668448766075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,511,0.052069331208864846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,1023,0.09052800138791402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,1023,0.09236799677213033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,2047,0.17004799842834473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,2047,0.17436800400416055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,4095,0.33186666170756024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,4095,0.3330346743265788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,1,0.013450667262077332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,1,0.009525333220760027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,8191,0.6668639977773031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,3,0.01392000044385592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,8191,0.6590559879938761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,3,0.009322666873534521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,7,0.01440000037352244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,7,0.00943999985853831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,15,0.017818666994571686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,15,0.009392000113924345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,31,0.01798933371901512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,31,0.009658666948477427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,63,0.018005333840847015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,63,0.009818666925032934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,127,0.021242665747801464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,16383,1.7401760419209797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,127,0.010741333166758219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,16383,1.8706506093343098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,255,0.015130666395028433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,255,0.011994666109482447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,511,0.015829333414634068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,511,0.012693333129088083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,1023,0.023391999304294586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,1023,0.015791999797026317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,2047,0.03411199897527695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,8191,0.07497600217660268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,2047,0.02657066782315572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,4095,0.05239999790986379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,1,0.010741333166758219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,4095,0.03180266668399175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,8191,0.05605866511662801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,1,0.007429333403706551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,7,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,16383,0.11286399761835735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,3,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,3,0.007178666690985362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,16383,0.08653866251309712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,7,0.007167999943097432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,15,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,15,0.00749333327015241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,31,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,31,0.007631999750932057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,63,0.010693332801262537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,63,0.007151999821265538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,511,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,127,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,127,0.007797333101431529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,1023,0.018976000448067982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,255,0.013962666193644205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,255,0.009535999968647957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,4095,0.06834133466084798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,511,0.019946667055288952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,1023,0.028463999430338543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,2047,0.0452159990866979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,2047,0.023205332458019257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,4095,0.03598399957021078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,8191,0.08727999528249104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,16383,0.13075733184814453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,8191,0.052095999320348106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,16383,0.07559999823570251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,float16,1,1.2065333525339763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,fp8,1,1.2316640218098958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,float16,3,1.2506026426951091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,fp8,3,1.275818665822347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,float16,7,1.3089333375295003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,fp8,7,1.3416320482889812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,float16,15,1.6614452997843425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,fp8,15,1.6673386891682942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,float16,31,1.66593599319458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,fp8,31,1.6576639811197917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,float16,63,1.679093360900879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,fp8,1,1.264837344487508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,float16,1,1.2268640200297039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,fp8,63,1.6724640528361003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,float16,3,1.2546933492024739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,fp8,3,1.2894666989644368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,float16,7,1.321349302927653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,fp8,7,1.3459572792053223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,float16,15,1.680448055267334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,fp8,15,1.6866186459859211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,float16,31,1.6895626386006672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,fp8,31,1.678069273630778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,float16,63,1.7026346524556477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,float16,1,0.23453332980473837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,fp8,1,0.17738133668899536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,fp8,3,0.17832533518473306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,float16,3,0.2348853349685669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,float16,15,0.23402667045593262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,float16,7,0.2347360054651896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,fp8,7,0.1781066656112671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,fp8,63,1.7055199940999348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,fp8,15,0.17723200718561807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,float16,31,0.23452266057332358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,fp8,31,0.17850132783253989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,float16,63,0.23296000560124716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,fp8,63,0.17679999272028604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,float16,1,0.3269600073496501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,fp8,1,0.15609600146611533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,fp8,3,0.1556106706460317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,float16,3,0.32941333452860516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,float16,7,0.3280426661173503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,fp8,7,0.15707733233769736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,fp8,15,0.15530133247375488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,float16,15,0.3254133264223735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,float16,31,0.3264106710751851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,fp8,31,0.1564853290716807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,float16,63,0.32316799958546955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,fp8,63,0.15606932838757834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,float16,1,2.4068214098612466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,fp8,1,2.4715147018432617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,float16,3,2.494080066680908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,fp8,3,2.54477326075236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,float16,7,2.6109493573506675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,fp8,7,2.658191998799642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,float16,15,3.316831906636556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,fp8,15,3.3046347300211587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,float16,1,2.448549270629883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,float16,31,3.349146525065104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,fp8,1,2.504800001780192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,fp8,31,3.30839474995931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,float16,3,2.5033013025919595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,fp8,3,2.552618662516276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,float16,1,0.4617813428243001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,float16,7,2.638490676879883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,fp8,7,2.68451722462972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,fp8,1,0.34745601812998456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,float16,15,3.37938658396403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,fp8,15,3.3420801162719727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,float16,31,3.372543970743815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,float16,3,0.46025065581003827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,fp8,31,3.3496265411376953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,fp8,3,0.348410685857137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,float16,7,0.4561920166015625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,fp8,7,0.3474666674931844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,float16,15,0.4607199827829997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,fp8,15,0.3469173510869344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,fp8,31,0.34914668401082355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,float16,31,0.4556746482849121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,float16,1,0.6451679865519205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,fp8,1,0.301530659198761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,fp8,3,0.30054932832717896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,float16,3,0.6439573367436727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,1,0.02294933299223582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,1,0.02351466566324234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,3,0.023775999744733173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,3,0.024330665667851765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,fp8,7,0.30108799537022907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,7,0.024693332612514496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,7,0.025040000677108765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,float16,7,0.6412853399912516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,fp8,15,0.3027893304824829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,float16,15,0.646511991818746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,15,0.030943999687830608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,15,0.031002665559450786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,fp8,31,0.3017653425534566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,31,0.03124266614516576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,float16,31,0.6416586637496948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,31,0.031018666923046112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,63,0.031317333380381264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,127,0.03761066744724909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,63,0.031162666777769726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,511,0.09612266222635905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,127,0.037477334340413414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,255,0.05700799822807312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,255,0.056746666630109154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,511,0.09538132945696513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,1023,0.17223467429478964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,1023,0.1744426687558492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,2047,0.32361066341400146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,2047,0.32928532361984253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,4095,0.6328853368759155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,1,0.023007998863856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,4095,0.6389866669972738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,1,0.023562667270501454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,3,0.02383466561635335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,3,0.02438933402299881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,7,0.024725332856178284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,8191,1.2791840235392253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,7,0.02515733242034912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,8191,1.2668639818827312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,63,0.03137599925200144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,15,0.031152000029881794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,15,0.031008000175158184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,31,0.03162133445342382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,31,0.031013332307338715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,63,0.031514666974544525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,127,0.03758399933576584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,127,0.03756266583998998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,255,0.05705066521962484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,255,0.0569706658522288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,511,0.09548800190289815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,511,0.09655466675758362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,1023,0.17280532916386923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,1023,0.1742453376452128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,2047,0.32760000228881836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,2047,0.32898133993148804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,4095,0.6485493183135986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,4095,0.6384906768798828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,1,0.012410666793584824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,1,0.009530666594703993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,3,0.012448000411192576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,3,0.009594666461149851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,7,0.012426666915416718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,8191,1.2876426378885906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,7,0.009546666716535887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,8191,1.2918559710184734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,15,0.01228800043463707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,15,0.009408000235756239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,127,0.01028266673286756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,255,0.015978666643301647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,31,0.012495999534924826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,31,0.00956266683836778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,63,0.012341332932313284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,63,0.009338666374484697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,127,0.012389333297808966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,255,0.012293333808581034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,511,0.023647998770078022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,511,0.01441066712141037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,1023,0.03458133339881897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,1023,0.02640533447265625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,2047,0.05492266515890757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,2047,0.036714665591716766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,4095,0.08533866206804912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,1,0.0074506668994824094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,4095,0.06611733138561249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,8191,0.10938133796056111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,8191,0.08332799871762593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,7,0.007162666569153468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,15,0.007477333148320516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,1,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,3,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,3,0.0074506668994824094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,7,0.011328000575304031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,63,0.0074560002734263735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,15,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,31,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,31,0.0074453335255384445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,63,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,511,0.0186666672428449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,127,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,127,0.0080960001796484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,255,0.021509334444999695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,255,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,511,0.028981332977612812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,1023,0.04751466711362203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,1023,0.027456000447273254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,2047,0.06663999954859416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,2047,0.03734933336575826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,4095,0.08695466319719951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,4095,0.05208000044027964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,8191,0.1276800036430359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,1,0.03294933338960012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,8191,0.07707733412583669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,1,0.03299200038115183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,1,0.033013333876927696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,1,0.033045334120591484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,3,0.03293866664171219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,3,0.03283733377854029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,15,0.03537066777547201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,3,0.03310933212439219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,3,0.033157333731651306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,7,0.03410666684309641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,7,0.03429333368937174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,7,0.034101332227389015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,7,0.03414933383464813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,15,0.0352960005402565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,15,0.0354720006386439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,15,0.03550933301448822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,31,0.043920000394185386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,31,0.04384533564249674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,31,0.04418133199214935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,31,0.04414399961630503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,63,0.04444266855716705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,63,0.04436799883842468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,63,0.04466133316357931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,63,0.044591998060544334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,127,0.04508799811204275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,127,0.04491733511288961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,127,0.04502399762471517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,127,0.04508799811204275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,255,0.045184001326560974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,255,0.05409599840641022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,255,0.045114666223526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,255,0.053914666175842285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,511,0.04571733375390371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,511,0.08238933483759563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,511,0.045184001326560974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,511,0.08398933211962382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,2047,0.25171200434366864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,1023,0.04553600152333578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,1023,0.13884799679120383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,1023,0.04535999894142151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,1023,0.14147733648618063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,2047,0.04555733501911163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,2047,0.045365333557128906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,4095,0.04528533418973287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,4095,0.4869439999262492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,2047,0.25668267409006756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,4095,0.04513066510359446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,128,1,float16,float16,1,0.03291733314593633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,4095,0.4773973226547241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,float16,1,0.03307733436425527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,128,1,float16,fp8,1,0.033215999603271484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,fp8,1,0.03309866786003113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,128,1,float16,float16,3,0.033173332611719765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,fp8,7,0.03422400106986364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,float16,3,0.0329120010137558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,128,1,float16,fp8,3,0.03301866600910822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,fp8,3,0.03305066625277201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,128,1,float16,float16,7,0.03429333368937174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,float16,7,0.03440533330043157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,128,1,float16,fp8,7,0.03428266694148382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,128,1,float16,float16,15,0.03557866563399633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,float16,15,0.035317334036032356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,128,1,float16,fp8,15,0.03549866626660029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,fp8,15,0.03540800015131632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,128,1,float16,float16,31,0.04409066836039225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,float16,31,0.044218664367993675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,128,1,float16,fp8,31,0.044256001710891724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,fp8,31,0.04444266855716705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,128,1,float16,float16,63,0.04442666471004486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,float16,63,0.044480000933011375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,128,1,float16,fp8,63,0.044879997769991554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,fp8,63,0.044922664761543274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,128,1,float16,float16,127,0.04532266656557719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,float16,127,0.04500266909599304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,128,1,float16,fp8,127,0.04524800181388855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,fp8,127,0.0450133333603541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,128,1,float16,float16,255,0.04518933097521464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,float16,255,0.054341331124305725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,128,1,float16,fp8,255,0.04524800181388855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,fp8,255,0.054197331269582115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,128,1,float16,float16,511,0.04548266530036926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,float16,511,0.08252800007661183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,128,1,float16,fp8,511,0.04509866734345754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,fp8,511,0.08354133367538452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,128,1,float16,float16,1023,0.045498669147491455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,float16,1023,0.13914133111635843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,128,1,float16,fp8,1023,0.04530133306980133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,fp8,1023,0.14196800192197165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,128,1,float16,float16,2047,0.04534933467706045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,128,1,float16,fp8,2047,0.045824001232783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,float16,2047,0.25180800755818683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,fp8,2047,0.2573866645495097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,128,1,float16,float16,4095,0.045461331804593406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,128,1,float16,fp8,4095,0.045141334335009255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,float16,4095,0.48574399948120117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,fp8,4095,0.48845334847768146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,3,0.011802667131026586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,3,0.01210133358836174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,1,0.011754666765530905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,1,0.01146666705608368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,7,0.011866666376590729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,7,0.011711999773979187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,15,0.011898666620254517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,15,0.011690666278203329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,31,0.011744000017642975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,31,0.011514666179815928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,63,0.011551999797423681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,127,0.011493333925803503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,63,0.011503999431928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,127,0.011434666812419891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,255,0.020986666282018025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,255,0.020746666938066483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,511,0.026208000878492992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,511,0.029002666473388672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,1023,0.03850133220354716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,1023,0.046949331959088646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,2047,0.06088533500830332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,2047,0.03997866561015447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,4095,0.040394666294256844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,4095,0.07701333363850911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,1,0.014592000593741735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,1,0.014741333822409311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,1,0.006709333509206772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,1,0.006927999978264173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,3,0.014842666685581207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,3,0.014741333822409311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,3,0.006768000001708667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,3,0.006714666883150737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,7,0.01461333284775416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,7,0.015263999501864115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,7,0.006741333131988843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,7,0.006751999879876773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,15,0.014677333335081736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,15,0.014661333213249842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,15,0.006784000123540561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,15,0.006890666360656421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,31,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,31,0.014826666563749313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,31,0.0069759997228781385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,31,0.007050666958093643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,63,0.014805333067973455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,63,0.01471466695268949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,63,0.00702400008837382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,63,0.007071999832987785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,127,0.014746667196353277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,127,0.014720000326633453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,127,0.007754666730761528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,127,0.007743999982873599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,255,0.026837334036827087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,255,0.026789332429567974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,255,0.009237333511312803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,255,0.010533332824707031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,511,0.036490666369597115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,511,0.04083200047413508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,511,0.010543999572594961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,511,0.018239999810854595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,1023,0.036576000352700554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,1023,0.05087999999523163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,1023,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,1023,0.02306666721900304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,2047,0.03910933434963226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,2047,0.06786666810512543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,2047,0.01062400018175443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,2047,0.033039999504884086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,4095,0.03888533264398575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,4095,0.09055999914805095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,4095,0.010618666807810465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,4095,0.05403733253479004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,1,0.007637333124876022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,1,0.007717333113153775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,1,0.008837333569924036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,1,0.008890666688481966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,3,0.007717333113153775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,3,0.007701333612203598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,3,0.008879999940594038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,3,0.008912000184257826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,7,0.007743999982873599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,7,0.007663999994595845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,7,0.008885333314538002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,7,0.008992000172535578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,15,0.007658666620651881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,15,0.007621333623925845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,15,0.008821333448092142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,15,0.00884799969693025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,31,0.008005333443482717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,31,0.007957333077987036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,31,0.009152000149091085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,31,0.009189333145817121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,63,0.008725333337982496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,63,0.008757333581646284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,63,0.00978133330742518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,63,0.009770666559537252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,127,0.008901333436369896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,127,0.008885333314538002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,127,0.011365332951148352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,127,0.011402666568756104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,255,0.008933333059151968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,255,0.008997333546479544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,255,0.011391999820868174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,255,0.011413333316644033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,511,0.008863999818762144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,511,0.009866666669646898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,511,0.011306667079528173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,511,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,1023,0.021573332448800404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,1023,0.0258240004380544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,1023,0.021589333812395733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,1023,0.025642665723959606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,2047,0.0215786670645078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,2047,0.029829333225886028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,2047,0.021594665944576263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,2047,0.029509333272775013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,4095,0.021615999440352123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,4095,0.040031999349594116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,4095,0.02163733293612798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,4095,0.03976533313592275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,8191,0.02179733415444692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,8191,0.05995733539263407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,8191,0.021674667795499165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,8191,0.05982399980227152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,16383,0.02181333303451538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,16383,0.09969600041707356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,16383,0.021583999196688335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,16383,0.0990666647752126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,128,1,float16,float16,1,0.008031999692320824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,float16,1,0.007754666730761528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,128,1,float16,fp8,1,0.00890666681031386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,fp8,1,0.00898133342464765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,128,1,float16,float16,3,0.007791999727487564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,float16,3,0.01044800008336703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,128,1,float16,fp8,3,0.009434666484594345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,fp8,3,0.009183999771873156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,128,1,float16,float16,7,0.007823999971151352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,float16,7,0.007903999959429106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,128,1,float16,fp8,7,0.008976000050703684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,fp8,7,0.009061333412925402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,128,1,float16,float16,15,0.007893333211541176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,float16,15,0.00797333319981893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,128,1,float16,fp8,15,0.009061333412925402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,fp8,15,0.009119999905427298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,128,1,float16,float16,31,0.008112000301480293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,float16,31,0.008053333188096682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,128,1,float16,fp8,31,0.009301333377758661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,fp8,31,0.009269333134094873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,128,1,float16,float16,63,0.008757333581646284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,float16,63,0.00874133345981439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,128,1,float16,fp8,63,0.009893333539366722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,fp8,63,0.009904000287254652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,128,1,float16,float16,127,0.008752000207702318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,float16,127,0.008842666943868002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,128,1,float16,fp8,127,0.011391999820868174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,fp8,127,0.011365332951148352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,128,1,float16,float16,255,0.008885333314538002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,float16,255,0.008863999818762144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,128,1,float16,fp8,255,0.011343999455372492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,fp8,255,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,128,1,float16,float16,511,0.008885333314538002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,float16,511,0.009824000298976898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,128,1,float16,fp8,511,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,fp8,511,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,128,1,float16,float16,1023,0.02170666555563609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,float16,1023,0.025663999219735462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,128,1,float16,fp8,1023,0.02162133405605952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,fp8,1023,0.025466665625572205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,128,1,float16,float16,2047,0.02170666555563609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,float16,2047,0.029989334444204967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,128,1,float16,fp8,2047,0.021776000658671062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,fp8,2047,0.02994133283694585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,128,1,float16,float16,4095,0.02182399978240331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,float16,4095,0.040031999349594116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,128,1,float16,fp8,4095,0.02160000056028366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,fp8,4095,0.03984000037113825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,128,1,float16,float16,8191,0.021770666042963665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,float16,8191,0.05986666679382324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,128,1,float16,fp8,8191,0.02161066730817159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,fp8,8191,0.05971199770768484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,128,1,float16,float16,16383,0.021695998807748158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,float16,16383,0.09964799880981445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,128,1,float16,fp8,16383,0.021776000658671062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,fp8,16383,0.09922666351000468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,1,0.008069333309928576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,1,0.008240000034372011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,3,0.007882666463653246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,3,0.007813333223263422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,7,0.007903999959429106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,7,0.007813333223263422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,15,0.007994666695594788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,15,0.00784533346692721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,31,0.008143999924262365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,31,0.007989333321650824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,63,0.008997333546479544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,63,0.008933333059151968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,127,0.00898133342464765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,127,0.008912000184257826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,255,0.009039999917149544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,255,0.008896000062425932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,511,0.00895999992887179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,511,0.009957333405812582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,1023,0.021770666042963665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,1023,0.025813333690166473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,2047,0.013264000415802002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,2047,0.01413333291808764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,4095,0.013823999712864557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,4095,0.01471466695268949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,8191,0.01461333284775416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,8191,0.017386666188637417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,16383,0.015413332730531693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,16383,0.01876266673207283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,1,0.00786666696270307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,1,0.007776000226537387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,1,0.006341333190600078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,1,0.006309333567818006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,3,0.007674666742483775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,3,0.007770666852593422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,3,0.007029333462317784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,3,0.006634666894872983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,7,0.007920000081261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,7,0.00789866658548514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,7,0.006906666482488315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,7,0.006560000280539195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,15,0.007946666950980822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,15,0.007818666597207388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,15,0.006704000135262807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,15,0.00666133314371109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,31,0.008314666648705801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,31,0.008058666562040647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,31,0.006757333253820737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,31,0.0064106664309899015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,63,0.008746666833758354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,63,0.008687999720374743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,63,0.006506666541099548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,63,0.0064266665528217954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,127,0.009008000294367472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,127,0.008826666822036108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,127,0.007157333195209503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,127,0.007050666958093643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,255,0.008954666554927826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,255,0.008816000074148178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,255,0.007813333223263422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,255,0.008469333251317343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,511,0.009109333157539368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,511,0.01002133327225844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,511,0.008005333443482717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,511,0.009066666786869368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,1023,0.012063999970753988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,1023,0.012069333344697952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,1023,0.007663999994595845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,1023,0.009759999811649323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,2047,0.012304000556468964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,2047,0.013034666577974955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,2047,0.007578666632374127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,2047,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,4095,0.01249066616098086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,4095,0.013669333110253016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,4095,0.007685333490371704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,4095,0.012458667159080505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,8191,0.01584533353646596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,8191,0.02056533346573512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,8191,0.007658666620651881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,8191,0.016143999993801117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,16383,0.01670933390657107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,16383,0.024175999065240223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,16383,0.007850666840871176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,1,0.008618666479984919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,16383,0.019285333653291065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,1,0.0085333331177632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,1,0.008565333361426989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,1,0.008485333373149237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,3,0.008709333216150602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,3,0.008517333616813024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,3,0.008597333605090777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,3,0.008463999877373377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,7,0.00884799969693025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,7,0.008816000074148178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,7,0.008746666833758354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,7,0.008682666967312494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,15,0.00892800030608972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,15,0.008703999842206636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,15,0.008693333094318708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,15,0.008602666358153025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,31,0.009375999992092451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,31,0.009365333244204521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,31,0.009418666362762451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,31,0.009232000137368837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,63,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,63,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,63,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,63,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,127,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,127,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,127,0.01126933346192042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,127,0.011306667079528173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,255,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,255,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,255,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,255,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,511,0.01126933346192042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,511,0.012810666114091873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,511,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,511,0.012730666746695837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,1023,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,1023,0.017765333255132038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,1023,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,1023,0.017818666994571686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,2047,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,2047,0.027978666126728058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,2047,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,2047,0.028762665887673695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,4095,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,4095,0.048122664292653404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,4095,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,4095,0.05005866785844167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,8191,0.011424000064531961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,8191,0.08750933408737183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,8191,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,8191,0.09181333581606548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,16383,0.16683199008305868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,16383,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,16383,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,16383,0.1734293301900228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,128,1,float16,float16,1,0.008912000184257826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,float16,1,0.008842666943868002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,128,1,float16,fp8,1,0.008650666723648706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,fp8,1,0.008485333373149237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,128,1,float16,float16,3,0.00873066671192646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,float16,3,0.008602666358153025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,fp8,3,0.008613333106040955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,128,1,float16,float16,7,0.008725333337982496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,float16,7,0.008912000184257826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,128,1,float16,fp8,7,0.008661333471536636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,fp8,7,0.00871999996403853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,128,1,float16,float16,15,0.00867733359336853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,float16,15,0.008805333326260248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,128,1,float16,fp8,15,0.008645333349704742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,fp8,15,0.008693333094318708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,128,1,float16,float16,31,0.00943999985853831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,float16,31,0.009461333354314169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,128,1,float16,fp8,31,0.009328000247478485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,fp8,31,0.009557333464423815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,128,1,float16,float16,63,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,float16,63,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,128,1,float16,fp8,63,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,128,1,float16,fp8,3,0.008709333216150602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,fp8,63,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,128,1,float16,float16,127,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,float16,127,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,128,1,float16,fp8,127,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,fp8,127,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,float16,255,0.011333333949247995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,128,1,float16,float16,255,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,128,1,float16,fp8,255,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,fp8,255,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,128,1,float16,float16,511,0.011338666081428528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,float16,511,0.012842666357755661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,128,1,float16,fp8,511,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,fp8,511,0.012960000584522883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,128,1,float16,float16,1023,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,float16,1023,0.017887999614079792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,128,1,float16,fp8,1023,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,fp8,1023,0.017818666994571686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,128,1,float16,float16,2047,0.011381333072980246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,float16,2047,0.028149334092934925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,128,1,float16,fp8,2047,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,fp8,2047,0.028794666131337483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,128,1,float16,float16,4095,0.011322667201360067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,float16,4095,0.04755199948946635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,128,1,float16,fp8,4095,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,float16,8191,0.08694932858149211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,128,1,float16,fp8,8191,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,fp8,8191,0.09170132875442505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,128,1,float16,float16,16383,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,float16,16383,0.16528000434239706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,128,1,float16,fp8,16383,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,fp8,16383,0.17393600940704346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,128,1,float16,float16,8191,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,fp8,4095,0.049770668148994446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,1,0.008613333106040955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,1,0.00879466657837232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,3,0.008512000242869059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,3,0.008629333227872849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,7,0.008703999842206636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,7,0.008912000184257826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,15,0.008581333483258883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,15,0.008837333569924036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,31,0.009402666861812273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,31,0.009365333244204521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,63,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,63,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,127,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,127,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,255,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,255,0.011343999455372492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,511,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,511,0.012896000097195307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,1023,0.013221333424250284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,1023,0.013248000293970108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,2047,0.014688000082969666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,2047,0.01443733274936676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,4095,0.014122666170199713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,4095,0.015119999647140503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,8191,0.017968000223239262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,8191,0.0232640008131663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,16383,0.028437333802382152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,16383,0.01933866615096728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,1,0.008656000097592672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,1,0.00860799973209699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,1,0.006362666686375936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,3,0.00847999999920527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,3,0.008693333094318708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,3,0.006693333387374878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,3,0.00660800002515316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,7,0.00874133345981439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,1,0.006437333300709724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,7,0.008778666456540426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,7,0.006565333033601443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,7,0.006474666918317477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,15,0.008938666433095932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,15,0.008656000097592672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,15,0.00697066696981589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,15,0.006538666784763336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,31,0.009578666960199675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,31,0.009408000235756239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,31,0.006645333642760913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,31,0.006544000158707301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,63,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,63,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,63,0.0064853330453236895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,63,0.006538666784763336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,127,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,127,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,127,0.007653333246707916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,127,0.007237333183487256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,255,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,255,0.011322667201360067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,255,0.007840000092983246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,255,0.008618666479984919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,511,0.011765333513418833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,511,0.011877333124478659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,511,0.008122666428486506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,511,0.008645333349704742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,1023,0.011802667131026586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,1023,0.012138667205969492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,1023,0.007717333113153775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,1023,0.009541333342591921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,2047,0.015247999380032221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,2047,0.01781333362062772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,2047,0.008058666562040647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,2047,0.011328000575304031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,4095,0.015856000284353893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,4095,0.019733333339293797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,4095,0.008581333483258883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,4095,0.013253333667914072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,8191,0.019424000134070713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,8191,0.027482666075229645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,8191,0.008101333553592363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,8191,0.017914666483799618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,16383,0.02250133454799652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,16383,0.038592000802357994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,16383,0.008021333565314611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,16383,0.025642665723959606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,1,0.060959999759991966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,1,0.0609440008799235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,1,0.06146133442719778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,1,0.060965334375699363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,3,0.060821334520975746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,3,0.06111466884613037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,3,0.060991997520128884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,3,0.06151466568311056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,7,0.0628959983587265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,7,0.06445333361625671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,7,0.0633546660343806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,7,0.06317866841952006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,15,0.06565333406130473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,15,0.06507733464241028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,15,0.06538133323192596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,15,0.06585066517194112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,31,0.08229333162307739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,31,0.08215466638406117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,31,0.08225599924723308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,31,0.08217066526412964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,63,0.08247999846935272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,63,0.0824480007092158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,63,0.0830080012480418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,63,0.08307200173536937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,127,0.08360532919565837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,127,0.08356799681981404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,127,0.08401599526405334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,127,0.08398399750391643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,255,0.08404266834259033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,255,0.10213866829872131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,255,0.08397866288820903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,255,0.1013866662979126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,511,0.08499200145403545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,511,0.15729066729545593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,511,0.08415466547012329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,511,0.1583146651585897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,1023,0.08467732866605122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,1023,0.0839359958966573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,1023,0.2667626738548279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,1023,0.271232008934021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,2047,0.08457600076993306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,2047,0.08413333694140117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,2047,0.48534401257832843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,128,1,float16,float16,1,0.060533334811528526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,2047,0.4935733477274577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,128,1,float16,fp8,1,0.06123200058937073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,float16,1,0.06052800019582113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,fp8,1,0.06109866499900818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,128,1,float16,float16,3,0.0606879989306132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,float16,3,0.060826669136683144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,128,1,float16,fp8,3,0.06092800199985504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,fp8,3,0.0610346645116806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,128,1,float16,float16,7,0.06287466486295064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,float16,7,0.06273066500822704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,128,1,float16,fp8,7,0.06330133477846782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,128,1,float16,float16,15,0.06514133512973785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,fp8,7,0.06318933268388112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,float16,15,0.06513066589832306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,128,1,float16,fp8,15,0.06570133566856384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,fp8,15,0.06532266736030579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,128,1,float16,float16,31,0.08182933429876964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,float16,31,0.08179200192292531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,128,1,float16,fp8,31,0.0823466678460439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,fp8,31,0.08232533435026805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,128,1,float16,float16,63,0.08272000153859456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,float16,63,0.08267733454704285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,128,1,float16,fp8,63,0.08334933718045552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,fp8,63,0.08333333333333333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,128,1,float16,float16,127,0.08381332953770955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,float16,127,0.08391466736793518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,128,1,float16,fp8,127,0.08396800359090169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,fp8,127,0.08404800295829773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,128,1,float16,float16,255,0.08493333061536153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,float16,255,0.10212799906730652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,128,1,float16,fp8,255,0.08417066931724548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,fp8,255,0.10154666503270467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,128,1,float16,float16,511,0.08480532964070638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,float16,511,0.1572053333123525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,128,1,float16,fp8,511,0.08428266644477844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,fp8,511,0.1590079963207245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,128,1,float16,float16,1023,0.08475200335184734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,float16,1023,0.26706133286158246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,128,1,float16,fp8,1023,0.08422933022181193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,fp8,1023,0.2722666660944621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,128,1,float16,float16,2047,0.084906667470932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,128,1,float16,fp8,2047,0.0842133363087972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,float16,2047,0.4957333405812581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,fp8,2047,0.4944746494293213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,1,0.01757866640885671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,1,0.017573333034912746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,3,0.017263999829689663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,3,0.01721599946419398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,7,0.017103999853134155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,7,0.017429333180189133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,15,0.01730666682124138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,15,0.01725333308180173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,31,0.017210666090250015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,31,0.01714666684468587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,63,0.01693333312869072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,63,0.016938666502634685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,127,0.01706133286158244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,127,0.01703466723362605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,255,0.03178133318821589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,255,0.03145066648721695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,511,0.04223466912905375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,511,0.048351998130480446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,1023,0.04234666625658671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,1023,0.05947199960549673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,2047,0.044549331068992615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,2047,0.07955733438332875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,1,0.023685333629449207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,1,0.023605334262053173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,1,0.007733333234985669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,1,0.007594666754206021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,3,0.02346133440732956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,3,0.024634666740894318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,3,0.007781333600481351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,3,0.007727999861041705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,7,0.023813332120577495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,7,0.0239680012067159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,7,0.007674666742483775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,7,0.00766933336853981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,15,0.02370133250951767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,15,0.023797333240509033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,15,0.007701333612203598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,15,0.007754666730761528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,31,0.02369600037733714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,31,0.02378133436044057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,31,0.0075573331365982694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,31,0.007733333234985669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,63,0.02333866556485494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,63,0.02342933416366577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,63,0.007525333513816197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,63,0.0075626665105422335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,127,0.023311999936898548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,127,0.023445333043734234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,127,0.009002666920423508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,127,0.009072000160813332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,255,0.04385599990685781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,255,0.04393066465854645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,255,0.015114666273196539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,255,0.017738666385412216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,511,0.044218664367993675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,511,0.05380799869696299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,511,0.015141333142916361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,511,0.0229120006163915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,1023,0.044549331068992615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,1023,0.06673066814740498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,1023,0.01526933287580808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,1023,0.03402133285999298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,2047,0.04731733103593191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,2047,0.09245333075523376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,2047,0.01525866612792015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,2047,0.05450133482615153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,1,0.013455999394257864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,1,0.013434667140245438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,1,0.013503999759753546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,1,0.01341333364446958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,3,0.013584000368913015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,3,0.013424000392357508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,3,0.013525333255529404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,3,0.013487999637921652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,7,0.013760000467300415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,7,0.013690666606028875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,7,0.013823999712864557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,7,0.0136266661187013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,15,0.013818666338920593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,15,0.01370666672786077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,15,0.013674666484196981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,15,0.01358933374285698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,31,0.013552000125249227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,31,0.013450667262077332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,31,0.013381333400805792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,31,0.01328533391157786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,63,0.015263999501864115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,63,0.01509333277742068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,63,0.01532799998919169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,63,0.015295999745527903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,127,0.015509333461523056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,127,0.015450666348139444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,127,0.015413332730531693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,127,0.015392000476519266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,255,0.015685333559910457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,255,0.01573333392540614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,255,0.015573333948850632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,255,0.015546667079130808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,511,0.015749332805474598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,511,0.020506666352351505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,511,0.015509333461523056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,511,0.017909333109855652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,1023,0.015754666179418564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,1023,0.025621332228183746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,1023,0.015498666713635126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,1023,0.025194667279720306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,2047,0.015696000307798386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,2047,0.040261333187421165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,2047,0.015562667200962702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,2047,0.04088533421357473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,4095,0.015802666544914246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,4095,0.06996266543865204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,4095,0.01563199982047081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,4095,0.07144000132878621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,8191,0.01600533351302147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,8191,0.12938666343688965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,8191,0.01552533358335495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,8191,0.1318666636943817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,16383,0.015834666788578033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,16383,0.24851200977961221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,16383,0.015829333414634068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,16383,0.2520800034205119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,128,1,float16,float16,1,0.01368533323208491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,float16,1,0.013610667238632837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,128,1,float16,fp8,1,0.013568000247081121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,fp8,1,0.013354666531085968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,128,1,float16,float16,3,0.0136266661187013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,float16,3,0.01358933374285698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,128,1,float16,fp8,3,0.013408000270525614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,fp8,3,0.013568000247081121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,128,1,float16,float16,7,0.013621332744757334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,float16,7,0.013776000589132309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,128,1,float16,fp8,7,0.0136266661187013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,fp8,7,0.013690666606028875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,128,1,float16,float16,15,0.013776000589132309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,float16,15,0.013669333110253016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,128,1,float16,fp8,15,0.01368533323208491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,fp8,15,0.01586666703224182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,128,1,float16,float16,31,0.013541333377361298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,float16,31,0.013546666751305262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,128,1,float16,fp8,31,0.013397333522637686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,fp8,31,0.01340266689658165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,128,1,float16,float16,63,0.015279999623696009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,float16,63,0.015311999867359797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,128,1,float16,fp8,63,0.015285332997639975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,fp8,63,0.015311999867359797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,128,1,float16,float16,127,0.015477333217859268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,float16,127,0.015557333827018738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,128,1,float16,fp8,127,0.015392000476519266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,fp8,127,0.01551466683546702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,128,1,float16,float16,255,0.01562133307258288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,float16,255,0.01573866605758667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,128,1,float16,fp8,255,0.015530666957298914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,fp8,255,0.015509333461523056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,128,1,float16,float16,511,0.015674666812022526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,float16,511,0.01842133328318596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,128,1,float16,fp8,511,0.01544533297419548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,fp8,511,0.01826133330663045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,128,1,float16,float16,1023,0.016565332810084026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,float16,1023,0.025989333788553875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,128,1,float16,fp8,1023,0.015482666591803232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,fp8,1023,0.025418666501839954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,128,1,float16,float16,2047,0.015722667177518208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,float16,2047,0.0405173326532046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,128,1,float16,fp8,2047,0.015477333217859268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,fp8,2047,0.040735999743143715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,128,1,float16,float16,4095,0.015610666324694952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,float16,4095,0.07027733325958252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,128,1,float16,fp8,4095,0.015461333096027374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,fp8,4095,0.07142400244871776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,128,1,float16,float16,8191,0.015647999942302704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,float16,8191,0.12965866923332214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,128,1,float16,fp8,8191,0.015471999843915304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,fp8,8191,0.13196800152460733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,128,1,float16,float16,16383,0.015685333559910457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,128,1,float16,fp8,16383,0.015376000354687372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,float16,16383,0.24835199117660522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,fp8,16383,0.2504319945971171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,1,0.0138026662170887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,1,0.013552000125249227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,3,0.013722666849692663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,7,0.013989333063364029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,7,0.013925333817799887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,3,0.013637332866589228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,15,0.013872000078360239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,15,0.013679999858140945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,31,0.013546666751305262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,31,0.013562666873137156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,63,0.015365333606799444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,63,0.015344000111023584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,127,0.015504000087579092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,127,0.015573333948850632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,255,0.015840000162522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,255,0.01580799991885821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,511,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,511,0.012991999586423239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,1023,0.013514666507641474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,1023,0.013162666310866674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,2047,0.017738666385412216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,2047,0.020501332978407543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,4095,0.01870399961868922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,4095,0.02277333289384842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,8191,0.022416000564893086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,8191,0.032672000428040825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,16383,0.025802666942278545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,16383,0.04569066564242045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,1,0.013669333110253016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,1,0.01421333352724711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,1,0.006544000158707301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,3,0.013471999516089758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,3,0.013631999492645264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,3,0.006746666505932808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,3,0.006640000268816948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,7,0.013834666460752487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,1,0.006549333532651265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,7,0.013861333330472311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,7,0.006741333131988843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,7,0.006549333532651265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,15,0.013962666193644205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,15,0.013616000612576803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,15,0.006634666894872983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,15,0.0064853330453236895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,31,0.013594667116800943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,31,0.013455999394257864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,31,0.006693333387374878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,31,0.006613333399097125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,63,0.015402667224407196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,63,0.015173333386580149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,63,0.007605333502093951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,63,0.006469333544373512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,127,0.015599999576807022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,127,0.015541333705186844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,127,0.007296000296870868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,127,0.007311999797821045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,255,0.012037333101034164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,255,0.01166933278242747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,255,0.007936000203092894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,255,0.00847999999920527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,511,0.011877333124478659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,511,0.012191999703645706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,511,0.007823999971151352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,511,0.008639999975760778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,1023,0.016794666647911072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,1023,0.01736533393462499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,1023,0.0081386665503184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,1023,0.010288000106811523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,2047,0.019285333653291065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,2047,0.02430933217207591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,2047,0.008373333141207695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,2047,0.012714666624863943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,4095,0.023797333240509033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,4095,0.032602667808532715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,4095,0.008090666805704435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,4095,0.016906666258970898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,8191,0.0281333327293396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,8191,0.04483733574549357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,8191,0.008474666625261307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,8191,0.027087998886903126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,16383,0.034490667283535004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,16383,0.06877333422501881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,16383,0.009183999771873156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,16383,0.037791999677817024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,float16,1,0.11529599626859029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,1,0.11435733238855998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,fp8,1,0.1160426636536916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,1,0.11532266934712727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,float16,3,0.11547733346621196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,3,0.11443199714024861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,fp8,3,0.11618666847546895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,3,0.11523733536402385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,float16,7,0.12037332852681477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,7,0.11929066975911458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,fp8,7,0.12136000394821167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,7,0.12011733651161194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,float16,15,0.12517866492271423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,15,0.12402133146921794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,fp8,15,0.12585066755612692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,15,0.12473066647847493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,float16,31,0.15981866916020712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,31,0.15942933162053427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,fp8,31,0.16037866473197937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,31,0.16038399934768677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,float16,63,0.16089600324630737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,63,0.1609653333822886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,fp8,63,0.16201600432395935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,63,0.16195199886957803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,float16,127,0.16310399770736694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,127,0.16339199741681418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,fp8,127,0.1639840006828308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,127,0.1637386679649353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,float16,255,0.1639253298441569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,255,0.19979733228683472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,fp8,255,0.16389333208402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,255,0.19830934206644693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,float16,511,0.16520532965660095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,fp8,511,0.1621280014514923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,511,0.30966933568318683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,511,0.30932799975077313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,float16,1023,0.16536532839139303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,fp8,1023,0.16370667020479837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,1023,0.5265013376871744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,64,128,1,float16,float16,1,0.11514666676521301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,1023,0.5388426780700684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,64,0,1,float16,float16,1,0.11487467090288798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,64,128,1,float16,fp8,1,0.11656000216801961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,64,0,1,float16,fp8,1,0.11559999982515971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,64,128,1,float16,float16,3,0.11610133449236552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,64,0,1,float16,float16,3,0.11508799592653911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,64,128,1,float16,fp8,3,0.11691199739774068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,64,0,1,float16,fp8,3,0.11582400401433308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,64,128,1,float16,float16,7,0.12122666835784912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,64,0,1,float16,float16,7,0.11937066912651062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,64,128,1,float16,fp8,7,0.12164266904195149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,64,0,1,float16,fp8,7,0.12036800384521484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,64,128,1,float16,float16,15,0.12566933035850525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,64,0,1,float16,float16,15,0.12426132957140605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,64,128,1,float16,fp8,15,0.12617066502571106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,64,0,1,float16,fp8,15,0.12499733765920003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,64,128,1,float16,float16,31,0.159770667552948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,64,0,1,float16,float16,31,0.1586079994837443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,64,128,1,float16,fp8,31,0.16077333688735962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,64,0,1,float16,fp8,31,0.1597599983215332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,64,0,1,float16,float16,63,0.16102932890256247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,64,128,1,float16,fp8,63,0.16115199526151022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,64,0,1,float16,fp8,63,0.1623840034008026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,64,0,1,float16,float16,127,0.16204266746838888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,64,128,1,float16,fp8,127,0.163674662510554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,64,0,1,float16,fp8,127,0.16250133514404297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,64,128,1,float16,float16,255,0.16409599781036377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,64,0,1,float16,float16,255,0.198362668355306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,64,128,1,float16,fp8,255,0.16387733817100525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,64,0,1,float16,fp8,255,0.19696533679962158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,64,128,1,float16,float16,511,0.16481600205103555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,64,128,1,float16,float16,63,0.15964800119400024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,64,0,1,float16,float16,511,0.3070720036824544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,64,128,1,float16,fp8,511,0.16386133432388306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,64,128,1,float16,float16,1023,0.1637279987335205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,64,0,1,float16,fp8,511,0.3111199935277303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,64,128,1,float16,float16,127,0.16201600432395935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,64,128,1,float16,fp8,1023,0.16318399707476297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,64,0,1,float16,float16,1023,0.5382186571756998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,64,0,1,float16,fp8,1023,0.5355573495229086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,1,0.028666667640209198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,float16,3,0.028330666323502857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,float16,1,0.028277332584063213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,3,0.028234665592511494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,float16,7,0.02829333394765854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,7,0.028421332438786823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,float16,15,0.028223998844623566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,15,0.028223998844623566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,float16,31,0.02865600089232127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,31,0.028725333511829376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,float16,63,0.028016000986099243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,63,0.027765333652496338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,float16,127,0.02789866675933202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,127,0.027866666515668232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,float16,255,0.05274133384227753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,255,0.052239999175071716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,float16,511,0.05347733199596405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,511,0.06461866696675618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,1023,0.08042133351167043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,float16,1023,0.05299200117588043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,float16,1,0.04138133426507314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,1,0.04248000184694926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,fp8,1,0.012485332787036896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,1,0.01249066616098086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,float16,3,0.04140799989302953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,3,0.04144533226887385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,fp8,3,0.012410666793584824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,3,0.012416000167528788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,float16,7,0.04162133236726125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,7,0.04151466737190882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,fp8,7,0.012357333054145178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,float16,15,0.04148799926042557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,15,0.04138133426507314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,fp8,15,0.012357333054145178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,15,0.012453333785136541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,float16,31,0.04165333261092504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,31,0.04178666571776072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,fp8,31,0.012576000144084295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,31,0.01259200026591619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,float16,63,0.041077333192030586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,63,0.04101866732041041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,fp8,63,0.012506666282812754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,63,0.012896000097195307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,float16,127,0.04140799989302953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,127,0.04128533353408178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,fp8,127,0.015263999501864115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,127,0.01482133318980535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,float16,255,0.04367466767628988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,255,0.043375998735427856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,fp8,255,0.014757333944241205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,255,0.020053333292404812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,float16,511,0.04359466830889384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,511,0.05659733215967814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,fp8,511,0.01481066644191742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,511,0.031002665559450786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,float16,1023,0.044293334086736046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,1023,0.07941866914431255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,7,0.012266666938861212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,fp8,1023,0.014917333920796713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,1023,0.05116266508897146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,float16,1,0.22428266207377115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,float16,1,0.22246400515238443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,fp8,1,0.22630399465560913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,fp8,1,0.2246560057004293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,float16,3,0.22438400983810425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,float16,3,0.22296533981959024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,fp8,3,0.22644267479578653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,fp8,3,0.22483734289805093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,float16,7,0.23478400707244873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,float16,7,0.23305066426595053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,fp8,7,0.2367146611213684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,fp8,7,0.2351306676864624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,float16,15,0.24447466929753622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,float16,15,0.24259199698766074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,fp8,15,0.24590933322906494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,fp8,15,0.24403733015060425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,float16,31,0.31302400430043537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,float16,31,0.31059734026590985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,fp8,31,0.3150026599566142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,fp8,31,0.3125600020090739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,float16,63,0.31542932987213135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,float16,63,0.3131093382835388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,fp8,63,0.3183199961980184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,fp8,63,0.3158773382504781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,float16,127,0.3197759985923767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,float16,127,0.31749866406122845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,fp8,127,0.3210933407147725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,fp8,127,0.3187626600265503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,float16,255,0.32173333565394086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,float16,255,0.3903306722640991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,fp8,255,0.32150399684906006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,fp8,255,0.3868480126063029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,64,128,1,float16,float16,1,0.2258346676826477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,64,0,1,float16,float16,1,0.22420267264048258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,64,128,1,float16,fp8,1,0.22737600406010947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,64,0,1,float16,fp8,1,0.22568533817927042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,64,128,1,float16,float16,3,0.22609599431355795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,64,0,1,float16,float16,3,0.22447466850280762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,64,128,1,float16,fp8,3,0.22752533356348673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,64,0,1,float16,fp8,3,0.22585066159566244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,64,128,1,float16,float16,7,0.23468265930811563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,64,0,1,float16,float16,7,0.23298132419586182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,64,128,1,float16,fp8,7,0.23694399992624918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,64,0,1,float16,fp8,7,0.23541865746180216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,64,128,1,float16,float16,15,0.2445919911066691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,64,0,1,float16,float16,15,0.24292800823847452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,64,128,1,float16,fp8,15,0.24593067169189453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,64,0,1,float16,fp8,15,0.2444480061531067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,64,128,1,float16,float16,31,0.3131840030352275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,64,0,1,float16,float16,31,0.311082661151886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,64,128,1,float16,fp8,31,0.31516265869140625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,64,0,1,float16,fp8,31,0.31297600269317627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,64,128,1,float16,float16,63,0.31568000713984173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,64,0,1,float16,float16,63,0.3134186665217082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,64,128,1,float16,fp8,63,0.31839466094970703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,64,0,1,float16,fp8,63,0.31627732515335083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,64,128,1,float16,float16,127,0.32049065828323364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,64,0,1,float16,float16,127,0.32043200731277466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,64,128,1,float16,fp8,127,0.321669340133667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,64,0,1,float16,fp8,127,0.3213599920272827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,64,128,1,float16,float16,255,0.321834663550059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,64,0,1,float16,float16,255,0.39349865913391113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,64,128,1,float16,fp8,255,0.32205333312352497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,64,0,1,float16,fp8,255,0.39052800337473553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,float16,1,0.051274667183558144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,float16,1,0.05124799907207489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,float16,3,0.05100266635417938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,float16,3,0.05091733237107595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,float16,7,0.05171733101209005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,float16,7,0.05158400038878123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,float16,15,0.050885334610939026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,float16,15,0.05062933266162872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,float16,31,0.05111999809741974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,float16,31,0.05082666873931885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,float16,63,0.05027733246485392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,float16,63,0.05009066561857859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,float16,127,0.05077866713205973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,float16,127,0.050810664892196655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,float16,255,0.05251200000445048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,float16,255,0.052186667919158936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,float16,1,0.07546666761239369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,float16,1,0.0758186678091685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,float16,3,0.07544533411661784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,float16,3,0.0758133331934611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,fp8,1,0.021327999730904896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,fp8,1,0.021242665747801464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,fp8,3,0.02130666623512904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,fp8,7,0.021397332350413006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,fp8,3,0.02145066608985265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,fp8,7,0.021381333470344543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,float16,15,0.07610666751861572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,float16,15,0.0757013310988744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,fp8,15,0.021488000949223835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,fp8,15,0.021189334491888683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,float16,31,0.07600533465544383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,float16,31,0.07577066620190938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,fp8,31,0.02141333371400833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,fp8,31,0.021375998854637146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,float16,63,0.07486400008201599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,float16,63,0.07436800003051758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,fp8,63,0.021210665504137676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,fp8,63,0.02107200026512146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,float16,127,0.0763679991165797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,float16,127,0.07600533465544383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,fp8,127,0.02624533325433731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,fp8,127,0.025850666066010792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,float16,7,0.07551466425259908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,float16,255,0.0812853326400121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,float16,7,0.0757013310988744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,float16,255,0.08038933575153351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,fp8,255,0.02568000058333079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,fp8,255,0.038592000802357994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,float16,1,0.43826667467753094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,float16,1,0.4413226842880249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,fp8,1,0.4429440100987752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,fp8,1,0.44309866428375244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,float16,3,0.43818668524424237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,float16,3,0.4418453375498454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,fp8,3,0.44278931617736816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,fp8,3,0.44314666589101154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,float16,7,0.4601866801579793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,float16,7,0.46350399653116864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,fp8,7,0.4649440050125122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,fp8,7,0.46457600593566895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,float16,15,0.4798613389333089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,float16,15,0.4832906723022461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,fp8,15,0.4830453395843506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,fp8,15,0.48284268379211426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,float16,31,0.6158506472905477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,float16,31,0.6204533179601034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,fp8,31,0.6198879877726237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,fp8,31,0.6195733149846395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,float16,63,0.6207466522852579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,float16,63,0.6254826784133911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,fp8,63,0.6267253160476685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,fp8,63,0.6263253291447958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,float16,127,0.6298826535542806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,float16,127,0.6351840098698934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,fp8,127,0.6325600147247314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,fp8,127,0.6323253313700358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,64,128,1,float16,float16,1,0.44248000780741376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,64,0,1,float16,float16,1,0.44598400592803955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,64,128,1,float16,fp8,1,0.446122686068217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,64,0,1,float16,fp8,1,0.445904016494751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,64,128,1,float16,float16,3,0.44620800018310547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,64,0,1,float16,float16,3,0.44300798575083417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,64,128,1,float16,fp8,3,0.44629867871602374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,64,0,1,float16,fp8,3,0.4492266575495402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,64,128,1,float16,float16,7,0.4603039820988973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,64,0,1,float16,float16,7,0.46026134490966797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,64,128,1,float16,fp8,7,0.46801598866780597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,64,0,1,float16,fp8,7,0.4647146860758464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,64,128,1,float16,float16,15,0.48017064730326336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,64,0,1,float16,float16,15,0.4838080008824666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,64,128,1,float16,fp8,15,0.48336533705393475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,64,0,1,float16,fp8,15,0.4833386739095052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,64,128,1,float16,float16,31,0.6208533445994059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,64,0,1,float16,float16,31,0.6161973476409912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,64,128,1,float16,fp8,31,0.6202773253122965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,64,0,1,float16,fp8,31,0.6246240139007568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,64,128,1,float16,float16,63,0.6211733420689901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,64,0,1,float16,float16,63,0.6212799946467081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,64,128,1,float16,fp8,63,0.6312160094579061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,64,0,1,float16,fp8,63,0.6270026763280233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,64,128,1,float16,float16,127,0.6397600173950195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,64,0,1,float16,float16,127,0.6407946745554606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,64,128,1,float16,fp8,127,0.6389813423156738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,64,0,1,float16,fp8,127,0.6327946583429972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,float16,1,0.0937013328075409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,float16,1,0.09344533085823059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,float16,3,0.09346666932106018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,float16,3,0.09409600496292114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,float16,7,0.09433600306510925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,float16,7,0.09387733538945515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,float16,15,0.09271466732025146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,float16,15,0.09270399808883667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,float16,31,0.09347200393676758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,float16,31,0.09286399682362874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,float16,63,0.09237333138783772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,float16,63,0.09227733810742696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,float16,127,0.09431466460227966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,float16,127,0.09294399619102478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,float16,1,0.14017599821090698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,float16,1,0.1402079959710439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,fp8,1,0.03845866769552231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,fp8,1,0.038480001191298165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,float16,3,0.1409119963645935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,float16,3,0.1405226687590281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,fp8,3,0.03846399982770284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,fp8,3,0.03842133283615112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,float16,7,0.14038933316866556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,float16,7,0.1405173341433207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,fp8,7,0.03857066730658213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,fp8,7,0.03858133405447006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,float16,15,0.14105066657066345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,float16,15,0.14070933063824972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,fp8,15,0.03867733230193456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,fp8,15,0.03869866579771042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,float16,31,0.14201600352923074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,float16,31,0.14198933045069376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,fp8,31,0.03858133405447006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,fp8,31,0.03866666555404663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,float16,63,0.14203199744224548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,float16,63,0.14174399773279825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,fp8,63,0.038378665844599404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,fp8,63,0.03846933444341024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,float16,127,0.14275200168291727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,float16,127,0.1426346699396769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,fp8,127,0.05345066885153452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,fp8,127,0.05329066514968872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,1,0.013072000195582708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,1,0.013151999562978745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,1,0.012768000364303589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,1,0.01267733300725619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,3,0.013162666310866674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,3,0.013162666310866674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,3,0.012778667112191519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,3,0.012618667135636011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,7,0.01340266689658165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,7,0.013530666629473368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,7,0.013125333935022354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,7,0.013007999708255133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,15,0.013280000537633896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,15,0.01320533330241839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,15,0.013269333789745966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,15,0.013221333424250284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,31,0.014709333578745524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,31,0.014767999450365702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,31,0.014778666198253632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,31,0.014720000326633453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,63,0.014885333677132925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,63,0.014959999670584997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,63,0.014805333067973455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,63,0.014752000570297241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,127,0.015040000279744467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,127,0.015002666662136713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,127,0.014938666174809137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,127,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,255,0.015216000378131866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,255,0.017498667041460674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,255,0.015066667149464289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,255,0.017301333447297413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,511,0.015018666783968607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,511,0.02473066747188568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,511,0.014906667172908783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,511,0.025034666061401367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,1023,0.01509333277742068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,1023,0.03962666789690653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,1023,0.01504533365368843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,1023,0.04079466561476389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,2047,0.014965333044528961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,2047,0.06945066650708516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,2047,0.014954666296641031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,2047,0.07197866837183635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,4095,0.015050667027632395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,4095,0.12851732969284058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,4095,0.01509333277742068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,4095,0.13369599978129068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,8191,0.015311999867359797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,8191,0.2539413372675578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,8191,0.015216000378131866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,8191,0.25683732827504474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,16383,0.015194666882356008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,16383,0.5545920133590698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,16383,0.015157333264748255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,16383,0.5822933514912924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,128,1,float16,float16,1,0.013338666409254074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,float16,1,0.013327999661366144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,128,1,float16,fp8,1,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,fp8,1,0.013077333569526672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,128,1,float16,float16,3,0.013237333546082178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,float16,3,0.013338666409254074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,128,1,float16,fp8,3,0.012842666357755661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,fp8,3,0.013013333082199097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,128,1,float16,float16,7,0.013690666606028875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,float16,7,0.01370666672786077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,128,1,float16,fp8,7,0.013338666409254074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,fp8,7,0.013359999905029932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,128,1,float16,float16,15,0.013173333058754602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,float16,15,0.01313599944114685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,128,1,float16,fp8,15,0.013045333325862885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,fp8,15,0.012837332983811697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,128,1,float16,float16,31,0.014592000593741735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,float16,31,0.014581333845853806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,128,1,float16,fp8,31,0.014720000326633453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,fp8,31,0.014709333578745524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,128,1,float16,float16,63,0.014864000181357065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,float16,63,0.014901333798964819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,128,1,float16,fp8,63,0.014912000546852747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,fp8,63,0.014938666174809137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,128,1,float16,float16,127,0.01509333277742068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,float16,127,0.015098666151364645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,128,1,float16,fp8,127,0.014959999670584997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,fp8,127,0.014933332800865173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,128,1,float16,float16,255,0.015135999768972397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,float16,255,0.017509333789348602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,128,1,float16,fp8,255,0.014778666198253632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,fp8,255,0.017301333447297413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,128,1,float16,float16,511,0.015050667027632395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,float16,511,0.02480533222357432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,128,1,float16,fp8,511,0.014981333166360855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,fp8,511,0.02515733242034912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,128,1,float16,float16,1023,0.015109332899252573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,float16,1023,0.0396373321612676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,128,1,float16,fp8,1023,0.015114666273196539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,fp8,1023,0.041146665811538696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,128,1,float16,float16,2047,0.015184000134468079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,float16,2047,0.06983466446399689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,128,1,float16,fp8,2047,0.015087999403476715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,fp8,2047,0.07205333312352498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,128,1,float16,float16,4095,0.015173333386580149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,float16,4095,0.12875733772913614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,128,1,float16,fp8,4095,0.01504533365368843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,fp8,4095,0.1334986686706543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,128,1,float16,float16,8191,0.01515199989080429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,float16,8191,0.252949337164561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,128,1,float16,fp8,8191,0.015114666273196539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,fp8,8191,0.2579360008239746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,128,1,float16,float16,16383,0.015200000256299973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,float16,16383,0.6682453155517578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,128,1,float16,fp8,16383,0.01498666654030482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,fp8,16383,0.5847733418146769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,1,0.013359999905029932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,1,0.013370666652917862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,3,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,3,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,7,0.013338666409254074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,7,0.013167999684810638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,15,0.01350933313369751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,15,0.01312000056107839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,31,0.014874666929244995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,31,0.01463466634353002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,63,0.014943999548753103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,63,0.014837333311637243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,127,0.015365333606799444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,127,0.015130666395028433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,255,0.012938667088747025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,255,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,511,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,511,0.012997332960367203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,1023,0.0194560003777345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,1023,0.018191999445358913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,2047,0.02120000123977661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,2047,0.026906666656335194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,4095,0.026362667481104534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,4095,0.03694933404525121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,8191,0.031008000175158184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,8191,0.05295466880003611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,16383,0.038389332592487335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,16383,0.07912533481915791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,1,0.010122666756312052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,1,0.0102186668664217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,1,0.006448000048597653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,1,0.007167999943097432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,3,0.009973333527644476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,3,0.010144000252087912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,3,0.006442666674653689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,3,0.006565333033601443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,7,0.010314666976531347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,7,0.010442666709423065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,7,0.006261333202322324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,7,0.0064319999267657595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,15,0.010149333626031876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,15,0.010341333225369453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,15,0.006218666831652324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,15,0.006474666918317477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,31,0.010101333260536194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,31,0.010389333590865135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,31,0.006271999950210254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,31,0.006645333642760913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,63,0.010026666646202406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,63,0.010250666489203772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,63,0.00655466690659523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,63,0.006538666784763336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,127,0.0100853331387043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,127,0.010234666367371878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,127,0.007125333572427432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,127,0.00725333330531915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,255,0.0122079998254776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,255,0.01240533341964086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,255,0.007536000261704127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,255,0.008416000132759413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,511,0.016970666746298473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,511,0.01775466650724411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,511,0.008069333309928576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,511,0.00955200009047985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,1023,0.021941334009170532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,1023,0.025392000873883564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,1023,0.008277333031098047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,1023,0.012351999680201212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,2047,0.027189334233601887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,2047,0.036346666514873505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,2047,0.008869333192706108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,2047,0.01952533299724261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,4095,0.03425599883000056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,4095,0.053632001082102455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,4095,0.008938666433095932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,4095,0.025392000873883564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,8191,0.035402665535608925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,8191,0.06780800223350525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,8191,0.008869333192706108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,8191,0.03481066723664602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,16383,0.03402666747570038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,16383,0.0904373327891032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,16383,0.009077333534757296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,16383,0.056559999783833824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,float16,1,0.8684533437093099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,fp8,1,0.8786880175272623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,float16,1,0.8759786287943522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,fp8,1,0.8784000078837076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,float16,3,0.8772213459014893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,float16,3,0.8687466780344645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,fp8,3,0.885263999303182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,fp8,3,0.8777813116709391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,float16,7,0.9210453033447266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,fp8,7,0.9234399795532227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,float16,15,0.9648586908976237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,float16,15,0.9571093718210856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,float16,7,0.9212533632914225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,fp8,7,0.9303840001424154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,fp8,15,0.9684373537699381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,fp8,15,0.9611519972483317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,float16,31,1.2398933569590251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,fp8,31,1.244773308436076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,float16,31,1.2414560317993164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,fp8,31,1.235317309697469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,float16,63,1.2535466353098552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,float16,63,1.261898676554362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,fp8,63,1.2610613505045574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,fp8,63,1.2501333554585774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,64,0,1,float16,float16,1,0.8856853644053141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,64,128,1,float16,float16,1,0.8789386749267578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,64,128,1,float16,fp8,1,0.8934880097707113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,64,0,1,float16,fp8,1,0.8862400054931641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,64,128,1,float16,float16,3,0.8867680231730143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,64,0,1,float16,float16,3,0.8800106843312582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,64,128,1,float16,fp8,3,0.8940533002217611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,64,0,1,float16,fp8,3,0.8862079779307047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,64,128,1,float16,float16,7,0.9228053092956543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,64,0,1,float16,float16,7,0.9154986540476481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,64,128,1,float16,fp8,7,0.9314026832580566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,64,0,1,float16,fp8,7,0.9242133299509684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,64,128,1,float16,float16,15,0.9751893679300944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,64,0,1,float16,float16,15,0.966426690419515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,64,128,1,float16,fp8,15,0.9722560246785482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,64,0,1,float16,fp8,15,0.9643786748250326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,64,128,1,float16,float16,31,1.252618630727132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,64,0,1,float16,float16,31,1.2431146303812664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,64,128,1,float16,fp8,31,1.2519946893056233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,64,0,1,float16,fp8,31,1.24180801709493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,64,128,1,float16,float16,63,1.2815893491109211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,64,0,1,float16,float16,63,1.271888017654419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,64,128,1,float16,fp8,63,1.2758293151855469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,64,0,1,float16,fp8,63,1.2654666900634766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,float16,1,0.1776640017827352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,float16,1,0.17586666345596313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,float16,3,0.17806400855382284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,float16,3,0.17572800318400064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,float16,7,0.17596266667048135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,float16,7,0.17601066827774048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,float16,15,0.1764906644821167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,float16,15,0.17799466848373413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,float16,31,0.1797920068105062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,float16,31,0.1798400084177653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,float16,63,0.17575999101003012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,float16,63,0.17796266078948975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,float16,1,0.273087998231252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,float16,1,0.27347733577092487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,fp8,1,0.08366933465003967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,float16,3,0.2738186717033386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,fp8,1,0.08352532982826233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,float16,3,0.2744426727294922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,fp8,3,0.08347733815511067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,fp8,3,0.08381332953770955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,float16,7,0.2754666606585185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,float16,7,0.2752053340276082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,fp8,7,0.08356266220410664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,fp8,7,0.08357866605122884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,float16,15,0.27717334032058716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,float16,15,0.2770826617876689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,fp8,15,0.08360000451405843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,fp8,15,0.08379200100898743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,float16,31,0.2789813280105591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,float16,31,0.27846399943033856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,fp8,31,0.08346666892369588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,fp8,31,0.08368000388145447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,float16,63,0.2759466568628947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,float16,63,0.2762239972750346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,fp8,63,0.08271466692288716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,fp8,63,0.08272000153859456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,1,64,128,1,float16,float16,1,1.8211627006530762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,1,64,128,1,float16,fp8,1,1.830074628194173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,float16,1,1.8204959233601887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,fp8,1,1.8304425875345867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,1,64,128,1,float16,float16,3,1.824837366739909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,float16,3,1.8248319625854492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,1,64,128,1,float16,fp8,3,1.8338452974955242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,fp8,3,1.8347892761230469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,1,64,128,1,float16,float16,7,1.8796159426371257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,float16,7,1.8804052670796711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,1,64,128,1,float16,fp8,7,1.8864107131958008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,fp8,7,1.8872373898824055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,1,64,128,1,float16,float16,15,1.9709173838297527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,float16,15,1.9702240626017253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,1,64,128,1,float16,fp8,15,1.9810667037963867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,fp8,15,1.9807732899983723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,1,64,128,1,float16,float16,31,2.5016372998555503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,float16,31,2.5021920204162598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,1,64,128,1,float16,fp8,31,2.523418744405111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,fp8,31,2.5244906743367515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,2,64,128,1,float16,float16,1,1.861456076304118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,2,64,0,1,float16,float16,1,1.863034725189209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,2,64,128,1,float16,fp8,1,1.8689120610555012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,2,64,0,1,float16,fp8,1,1.8716266949971516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,2,64,128,1,float16,float16,3,1.8654613494873047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,2,64,0,1,float16,float16,3,1.8661173184712727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,2,64,128,1,float16,fp8,3,1.8753387133280437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,2,64,0,1,float16,fp8,3,1.8749972979227703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,2,64,128,1,float16,float16,7,1.910154660542806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,2,64,128,1,float16,fp8,7,1.9163947105407715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,2,64,0,1,float16,fp8,7,1.9162880579630535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,2,64,0,1,float16,float16,7,1.9099253018697102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,2,64,128,1,float16,float16,15,1.9960907300313313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,2,64,0,1,float16,float16,15,1.9966293970743816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,2,64,128,1,float16,fp8,15,2.006175994873047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,2,64,0,1,float16,fp8,15,2.0048747062683105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,2,64,0,1,float16,float16,31,2.5039572715759277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,2,64,128,1,float16,float16,31,2.507370630900065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,2,64,128,1,float16,fp8,31,2.529359976450602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,2,64,0,1,float16,fp8,31,2.5266613960266113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,4,64,128,1,float16,float16,1,0.3486400047938029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,float16,1,0.34918399651845294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,4,64,128,1,float16,float16,3,0.349834680557251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,float16,3,0.34942932923634845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,4,64,128,1,float16,float16,7,0.35283199946085614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,float16,7,0.35303465525309247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,4,64,128,1,float16,float16,15,0.34989333152770996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,float16,15,0.35042667388916016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,4,64,128,1,float16,float16,31,0.3497013250986735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,float16,31,0.3509440024693807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,8,64,128,1,float16,float16,1,0.5498026609420776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,8,64,128,1,float16,fp8,1,0.16109866897265115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,float16,1,0.5526133378346761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,fp8,1,0.15985600153605142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,8,64,128,1,float16,fp8,3,0.16004799803098044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,float16,3,0.5536426703135172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,fp8,3,0.16009066502253214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,8,64,128,1,float16,float16,7,0.549343983332316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,float16,7,0.5506506760915121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,fp8,7,0.1597493290901184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,8,64,128,1,float16,float16,3,0.5535146792729696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,8,64,128,1,float16,float16,15,0.5510506629943848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,float16,15,0.5511519908905029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,8,64,128,1,float16,fp8,15,0.15973333517710367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,fp8,15,0.16011200348536173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,8,64,128,1,float16,float16,31,0.5515893300374349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,float16,31,0.551962653795878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,8,64,128,1,float16,fp8,31,0.1595306694507599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,8,64,128,1,float16,fp8,7,0.15996799866358438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,1,0.01863466699918111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,1,0.0186666672428449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,1,0.020576000213623047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,1,0.018672000616788864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,3,0.018559999763965607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,3,0.018474667022625606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,fp8,31,0.15994667013486227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,3,0.01871466636657715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,3,0.018640000373125076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,7,0.019167999426523846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,7,0.01924266666173935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,7,0.019365333020687103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,15,0.01978133370478948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,15,0.01993600030740102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,15,0.019808000574509304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,15,0.019978666057189304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,31,0.024480000138282776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,31,0.02473066747188568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,7,0.019343999524911244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,31,0.02480533222357432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,31,0.024634666740894318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,63,0.025061334172884624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,63,0.02489600082238515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,63,0.02513599892457326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,63,0.025093334416548412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,127,0.0252960001428922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,127,0.025061334172884624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,127,0.025248001019159954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,127,0.02532800038655599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,255,0.025413334369659424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,255,0.030042665700117748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,255,0.025706666211287182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,255,0.030053332448005676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,511,0.025370667378107708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,511,0.04450133442878723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,511,0.02908266584078471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,511,0.0450133333603541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,1023,0.025514667232831318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,1023,0.025386666258176167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,1023,0.0730453332265218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,1023,0.07503999769687653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,2047,0.02590399980545044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,2047,0.13143466909726462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,2047,0.02569066733121872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,2047,0.13434132933616638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,4095,0.025920001169045765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,4095,0.24559466044108072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,4095,0.02593066543340683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,4095,0.2520586649576823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,8191,0.02569066733121872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,8191,0.025589334468046825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,8191,0.4826666514078776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,128,1,float16,float16,1,0.01882133384545644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,8191,0.48934932549794513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,float16,1,0.019130667050679524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,128,1,float16,fp8,1,0.018986667195955913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,fp8,1,0.018965333700180054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,128,1,float16,float16,3,0.019237333287795384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,float16,3,0.018874666343132656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,fp8,3,0.018853332847356796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,128,1,float16,fp8,3,0.019082666685183842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,128,1,float16,float16,7,0.019386666516462963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,float16,7,0.019296000401178997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,128,1,float16,fp8,7,0.01933866615096728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,fp8,7,0.019466667125622433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,128,1,float16,float16,15,0.019813333948453266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,float16,15,0.020026666422684986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,128,1,float16,fp8,15,0.01998399943113327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,fp8,15,0.020128000527620316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,128,1,float16,float16,31,0.02470933397610982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,float16,31,0.02476799984773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,128,1,float16,fp8,31,0.024656000236670177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,fp8,31,0.024864000578721363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,128,1,float16,float16,63,0.025034666061401367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,float16,63,0.025008000433444977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,128,1,float16,fp8,63,0.02499733368555705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,fp8,63,0.025237334271272022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,128,1,float16,float16,127,0.025231999655564625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,float16,127,0.025013332565625507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,128,1,float16,fp8,127,0.025258667767047882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,fp8,127,0.02515200028816859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,128,1,float16,float16,255,0.025370667378107708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,float16,255,0.029887999097506206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,128,1,float16,fp8,255,0.025360000630219776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,fp8,255,0.02977599948644638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,128,1,float16,float16,511,0.025301332275072735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,float16,511,0.0443146675825119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,128,1,float16,fp8,511,0.025237334271272022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,fp8,511,0.044853334625562034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,128,1,float16,float16,1023,0.025349333882331848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,float16,1023,0.07294400036334991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,128,1,float16,fp8,1023,0.025290665527184803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,fp8,1023,0.07487999896208446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,128,1,float16,float16,2047,0.025221332907676697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,128,1,float16,fp8,2047,0.025407999753952026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,float16,2047,0.13078932960828146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,fp8,2047,0.13412800431251526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,128,1,float16,float16,4095,0.025333332518736523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,128,1,float16,fp8,4095,0.025253333151340485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,float16,4095,0.2456373373667399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,fp8,4095,0.2528853416442871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,128,1,float16,float16,8191,0.025253333151340485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,128,1,float16,fp8,8191,0.025402667621771496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,1,0.011567999919255575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,float16,8191,0.49395732084910077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,1,0.011530666301647821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,fp8,8191,0.49048535029093426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,3,0.01145600030819575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,3,0.011472000430027643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,7,0.01139733319481214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,7,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,15,0.011413333316644033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,15,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,31,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,31,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,63,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,63,0.011322667201360067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,127,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,127,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,255,0.01331199953953425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,255,0.013616000612576803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,511,0.018976000448067982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,511,0.01958400011062622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,1023,0.02477866659561793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,1023,0.02868266652027766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,2047,0.031018666923046112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,2047,0.04130133241415024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,4095,0.038880000511805214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,4095,0.06172266602516174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,8191,0.038431999584039055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,8191,0.07811200122038524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,1,0.010533332824707031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,1,0.006560000280539195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,3,0.010373333469033241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,3,0.006634666894872983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,3,0.006677333265542984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,7,0.010661333799362183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,7,0.010522666076819101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,7,0.006517333288987477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,7,0.006741333131988843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,15,0.010634666929642359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,15,0.010570666442314783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,15,0.006522666662931442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,15,0.006511999915043513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,31,0.010629333555698395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,31,0.010549332946538925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,31,0.006528000036875407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,31,0.006586666529377301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,63,0.010634666929642359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,63,0.010698666175206503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,1,0.010410666465759277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,63,0.006624000146985054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,127,0.010693332801262537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,1,0.006784000123540561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,127,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,127,0.007178666690985362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,127,0.007242666557431221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,3,0.010522666076819101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,255,0.018474667022625606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,255,0.009093333035707474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,511,0.023391999304294586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,511,0.02532266577084859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,511,0.00919999989370505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,511,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,1023,0.033439998825391136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,63,0.006437333300709724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,1023,0.040031999349594116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,255,0.018565333137909572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,1023,0.00927466650803884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,1023,0.019029332945744198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,255,0.008346666892369589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,2047,0.052485331892967224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,2047,0.009429333110650381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,2047,0.03585066646337509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,2047,0.023621333142121632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,4095,0.03531199942032496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,4095,0.06679999828338623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,4095,0.009733333562811216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,4095,0.03346133232116699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,8191,0.03510933369398117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,8191,0.0888266662756602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,8191,0.009573333586255709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,8191,0.05453333258628845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,1,0.03234133372704188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,1,0.033173332611719765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,3,0.03389333436886469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,7,0.03499199946721395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,7,0.03552533437808355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,15,0.04428266485532125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,3,0.03483733286460241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,15,0.04388799766699473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,31,0.044591998060544334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,31,0.044031997521718345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,63,0.04525866607824961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,63,0.04448533554871877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,127,0.05385600030422211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,127,0.05418666700522105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,255,0.08257600168387096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,255,0.08252800007661183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,511,0.14082133769989014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,511,0.1407360037167867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,1023,0.2553013364473979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,1023,0.25760533412297565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,2047,0.48389331499735516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,2047,0.4851466814676921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,float16,1,0.03322133421897888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,4095,0.9560320377349854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,4095,0.9438292980194092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,fp8,1,0.03325333446264267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,float16,3,0.03401066611210505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,fp8,3,0.03451200077931086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,float16,7,0.03509866694609324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,fp8,7,0.035962666074434914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,float16,15,0.04437866806983948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,fp8,15,0.044213334719340004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,float16,31,0.045082668463389076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,fp8,31,0.04428266485532125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,float16,63,0.044863998889923096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,fp8,63,0.045194665590922035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,float16,127,0.05406400064627329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,fp8,127,0.053770666321118675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,float16,255,0.08377599716186523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,float16,511,0.14019200205802917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,fp8,255,0.08286400139331818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,fp8,511,0.14192533493041992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,float16,1023,0.2544586658477783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,fp8,1023,0.255568007628123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,float16,2047,0.4933760166168213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,fp8,2047,0.4851200183232625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,float16,4095,0.954960028330485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,fp8,4095,0.970474640528361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,3,0.012821332861979803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,7,0.012757333616415659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,1,0.013194666554530462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,15,0.013034666577974955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,31,0.012863999853531519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,63,0.012671999633312225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,127,0.01320533330241839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,255,0.02548266698916753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,511,0.0340639998515447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,2047,0.08077866832415263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,1023,0.05705599983533224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,4095,0.10736533006032307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,1,0.007717333113153775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,3,0.007631999750932057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,3,0.015722667177518208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,1,0.01588800052801768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,15,0.015642666568358738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,7,0.015504000087579092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,31,0.01573866605758667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,31,0.007749333356817563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,63,0.015696000307798386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,63,0.007637333124876022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,127,0.015573333948850632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,127,0.008586666857202848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,255,0.030394665896892548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,255,0.017231999586025875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,511,0.048026666045188904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,511,0.02480533222357432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,1023,0.06227200229962667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,2047,0.08614400029182434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,2047,0.04863466819127401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,4095,0.12813333670298258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,4095,0.07369066774845123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,7,0.007738666608929634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,15,0.007653333246707916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,1,0.008837333569924036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,3,0.008986666798591614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,7,0.008858666444818178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,1,0.008965333302815756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,7,0.009002666920423508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,3,0.008837333569924036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,15,0.009216000015536943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,15,0.009338666374484697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,31,0.009679999823371569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,31,0.010064000263810158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,63,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,63,0.011343999455372492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,127,0.011461333682139715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,127,0.01138666644692421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,255,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,255,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,511,0.025573333104451496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,511,0.025568000972270966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,1023,0.029626667499542236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,1023,0.02977066735426585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,2047,0.04008533308903376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,2047,0.04002666721741358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,4095,0.060191998879114784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,4095,0.05983466903368632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,8191,0.10063999891281128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,8191,0.09896000226338704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,16383,0.18046400944391885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,16383,0.17736534277598062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,1,0.00877333308259646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,fp8,1,0.008853333070874214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,3,0.008687999720374743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,fp8,3,0.008885333314538002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,7,0.008762666955590248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,fp8,7,0.008879999940594038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,15,0.009008000294367472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,fp8,15,0.009125333279371262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,31,0.00966933307548364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,fp8,31,0.009695999945203463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,63,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,fp8,63,0.01128000020980835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,127,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,fp8,127,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,255,0.011407999942700068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,fp8,255,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,fp8,511,0.02548266698916753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,511,0.025568000972270966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,1023,0.034245334565639496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,1023,0.029535998900731403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,fp8,1023,0.029877332349618275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,2047,0.03996799886226654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,4095,0.06043200194835663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,fp8,4095,0.059530665477116905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,fp8,2047,0.03988266736268997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,8191,0.1006773312886556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,fp8,8191,0.09922666351000468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,fp8,16383,0.1774453322092692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,16383,0.1803306738535563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,3,0.00873066671192646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,1,0.008965333302815756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,7,0.008869333192706108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,15,0.009119999905427298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,31,0.009775999933481216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,63,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,127,0.011525332927703857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,255,0.0116799995303154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,511,0.025733334322770435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,1023,0.029663999875386555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,2047,0.017407999684413273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,8191,0.02216533323129018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,4095,0.017973333597183228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,16383,0.0245919997493426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,1,0.006864000111818314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,1,0.008709333216150602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,3,0.008986666798591614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,3,0.006751999879876773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,7,0.008821333448092142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,7,0.006784000123540561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,15,0.009237333511312803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,15,0.006895999734600385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,31,0.009877333417534828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,31,0.006837333242098491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,63,0.011493333925803503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,63,0.006800000245372455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,127,0.01166933278242747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,127,0.007194666812817256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,255,0.011754666765530905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,255,0.00892800030608972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,511,0.025829332570234936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,511,0.009253333633144697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,1023,0.009866666669646898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,1023,0.013674666484196981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,2047,0.011306667079528173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,2047,0.015024000157912573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,4095,0.016602666427691776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,4095,0.012826666235923767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,8191,0.02605866640806198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,8191,0.017237332959969837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,16383,0.030896000564098358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,16383,0.02130666623512904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,1,0.00842666688064734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,1,0.008416000132759413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,3,0.008538666491707167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,3,0.008639999975760778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,7,0.0085333331177632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,7,0.008698666468262672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,15,0.009413333609700203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,15,0.009472000102202097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,31,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,31,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,63,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,63,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,127,0.011333333949247995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,127,0.011343999455372492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,255,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,255,0.01357866699496905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,511,0.01807466646035512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,511,0.018906666586796444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,1023,0.028325334191322327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,1023,0.02880000074704488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,2047,0.048207998275756836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,2047,0.0488319993019104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,4095,0.08782399694124858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,4095,0.08915199836095174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,8191,0.16675732533137003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,8191,0.1688213348388672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,16383,0.321397324403127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,16383,0.32807467381159466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,1,0.008197333042820295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,3,0.008336000144481659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,fp8,1,0.008890666688481966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,fp8,3,0.008538666491707167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,7,0.00884799969693025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,fp8,7,0.0084906667470932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,15,0.009546666716535887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,fp8,15,0.009093333035707474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,31,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,fp8,31,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,63,0.011503999431928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,fp8,63,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,127,0.011706666400035223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,fp8,127,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,255,0.013386666774749756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,fp8,255,0.012784000486135483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,511,0.018122666825850803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,fp8,511,0.017690667261679966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,1023,0.028581333657105763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,fp8,1023,0.027973333994547527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,2047,0.04836266736189524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,fp8,2047,0.04783466458320618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,4095,0.08796266714731853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,fp8,4095,0.08752533793449402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,fp8,8191,0.16647467017173767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,8191,0.16796799500783285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,16383,0.3248959978421529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,fp8,16383,0.32682667175928753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,1,0.008421333506703377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,3,0.008527999743819237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,7,0.008757333581646284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,15,0.009296000003814697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,31,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,63,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,127,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,255,0.012719999998807907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,511,0.017850667238235474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,1023,0.01578666642308235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,2047,0.018005333840847015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,4095,0.019621333728233974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,8191,0.03046933313210805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,16383,0.03738666574160258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,1,0.00847999999920527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,1,0.006789333497484525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,3,0.008496000121037165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,3,0.006746666505932808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,7,0.008458666503429413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,7,0.006858666737874349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,15,0.009413333609700203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,15,0.006826666494210561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,31,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,31,0.006853333363930385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,63,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,63,0.00697066696981589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,127,0.011429333438475927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,127,0.007354666789372762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,255,0.012831999609867731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,255,0.009061333412925402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,511,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,511,0.009429333110650381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,1023,0.010149333626031876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,2047,0.020432000358899433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,2047,0.012719999998807907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,4095,0.024256000916163128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,4095,0.014954666296641031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,8191,0.02037866661945979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,16383,0.053157334526379905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,16383,0.03527999917666117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,1,0.05991466840108236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,1,0.061477333307266235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,3,0.06222933530807495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,3,0.06331733365853627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,7,0.0650133341550827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,7,0.06600533425807953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,1023,0.013765333841244379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,15,0.08246399958928426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,31,0.08272000153859456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,31,0.08211733400821686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,63,0.08316266536712646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,63,0.08278400202592213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,15,0.08201066652933757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,127,0.10097600022951762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,127,0.10087466239929199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,255,0.15702933073043823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,255,0.15668800473213196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,511,0.2686240077018738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,511,0.2690666715304057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,1023,0.4907199939092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,1023,0.49105600516001385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,2047,0.9440373579661051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,2047,0.9361013571421305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,float16,1,0.060229331254959106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,fp8,1,0.06159999966621399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,float16,3,0.06221333146095276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,fp8,3,0.06323199967543285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,8191,0.03438399980465571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,float16,7,0.06498666604359944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,fp8,7,0.06599466502666473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,fp8,15,0.08228266735871632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,float16,31,0.08269333342711131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,fp8,31,0.08216533561547597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,float16,63,0.08333333333333333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,fp8,63,0.08309333523114522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,float16,127,0.10128000378608704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,fp8,127,0.10142399867375691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,float16,255,0.15708266695340475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,float16,15,0.08306133250395457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,fp8,255,0.1578986644744873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,float16,511,0.26901866992314655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,fp8,511,0.2707253297170003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,float16,1023,0.5001066525777181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,fp8,1023,0.49086932341257733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,1,0.019786667078733444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,float16,2047,0.9572800000508627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,fp8,2047,0.957482655843099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,3,0.01988799994190534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,7,0.01998399943113327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,15,0.01941866676012675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,31,0.019797333826621372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,63,0.01940800001223882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,127,0.01924266666173935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,255,0.037589333951473236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,511,0.05895466605822245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,1023,0.0771679977575938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,2047,0.1074026624361674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,1,0.02566933383544286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,1,0.012138667205969492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,3,0.02606933315594991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,3,0.012186666329701742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,7,0.025589334468046825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,7,0.012053333222866058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,15,0.025621332228183746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,15,0.012128000458081564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,31,0.025589334468046825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,31,0.012058666596810022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,63,0.025397333006064098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,63,0.012282667060693106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,127,0.026191999514897663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,127,0.014815999815861383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,255,0.05232533315817515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,255,0.025631998976071674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,511,0.06689600149790446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,511,0.03572800010442734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,1023,0.08637866377830505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,1023,0.050213331977526345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,2047,0.13402666648228964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,2047,0.07507733503977458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,1,0.013525333255529404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,1,0.013317332913478216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,3,0.013471999516089758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,3,0.01351999988158544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,7,0.013450667262077332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,7,0.013653332988421122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,15,0.013418667018413544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,15,0.013562666873137156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,31,0.015178666760524115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,31,0.015344000111023584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,63,0.01545599972208341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,63,0.01553600033124288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,127,0.015397333850463232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,127,0.015706667055686314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,255,0.018160000443458557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,255,0.018496000518401463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,511,0.02590399980545044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,511,0.025850666066010792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,1023,0.04028266668319702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,1023,0.04078399886687597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,2047,0.06986666719118755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,2047,0.07092800239721934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,4095,0.1292693316936493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,4095,0.13083199659983316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,8191,0.2489173412322998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,8191,0.24994667371114096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,16383,0.4873493512471517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,16383,0.48843733469645184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,1,0.013274667163689932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,fp8,1,0.013541333377361298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,3,0.013183999806642532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,fp8,3,0.013253333667914072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,7,0.013386666774749756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,fp8,7,0.013712000101804733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,15,0.01320533330241839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,fp8,15,0.013658666362365087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,31,0.015061333775520325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,fp8,31,0.01534933348496755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,63,0.015344000111023584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,fp8,63,0.015557333827018738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,127,0.015493333339691162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,fp8,127,0.015594666202863058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,255,0.01836799954374631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,fp8,255,0.018538666268189747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,511,0.02609066665172577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,fp8,511,0.02629866699377696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,1023,0.04058666775623957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,fp8,1023,0.04095999896526337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,2047,0.07044800122578938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,fp8,2047,0.0710399995247523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,4095,0.1309279998143514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,fp8,4095,0.13105600078900656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,8191,0.2508053382237752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,fp8,8191,0.2504640022913615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,16383,0.5328160127003988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,fp8,16383,0.48767999807993573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,1,0.013381333400805792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,3,0.013290667285521826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,7,0.013616000612576803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,15,0.013440000514189402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,31,0.015072000523408255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,63,0.015376000354687372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,127,0.01589866727590561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,255,0.018394666413466137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,511,0.015013333410024643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,1023,0.015557333827018738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,2047,0.02462399999300639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,4095,0.03027733415365219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,8191,0.043935999274253845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,16383,0.06468800206979115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,1,0.013365333278973898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,1,0.006709333509206772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,3,0.013162666310866674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,3,0.006784000123540561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,7,0.013434667140245438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,7,0.006645333642760913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,15,0.014362666755914688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,15,0.006895999734600385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,31,0.015216000378131866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,31,0.006815999746322632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,63,0.015589332828919092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,63,0.00679466687142849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,127,0.015770666301250458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,127,0.0074346667776505155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,255,0.00902399979531765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,255,0.012938667088747025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,511,0.013445333888133367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,511,0.009408000235756239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,1023,0.01952533299724261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,1023,0.011434666812419891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,2047,0.028399998943010967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,4095,0.041797334949175514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,4095,0.02314666658639908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,8191,0.059706668059031166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,8191,0.03205333401759466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,16383,0.09067199627558391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,16383,0.0536106675863266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,1,0.11460266510645549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,1,0.11660800377527873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,3,0.11890133221944173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,3,0.12113066514333089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,7,0.12453866998354594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,7,0.12665067116419473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,15,0.15962666273117065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,15,0.15920000274976095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,31,0.15997333327929178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,31,0.1590666671593984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,63,0.16128533085187277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,63,0.1606666644414266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,127,0.1962613264719645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,2047,0.014352000008026758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,127,0.19625065724054971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,255,0.3094453414281209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,255,0.3072320024172465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,511,0.533242662747701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,511,0.5292160113652548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,1023,0.986021359761556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,1023,0.9695040384928385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,float16,1,0.11466667056083679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,fp8,1,0.11687466502189636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,float16,3,0.11957866946856181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,fp8,3,0.12100266416867574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,float16,7,0.1253653367360433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,fp8,7,0.1266986628373464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,float16,15,0.16090666254361471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,fp8,15,0.15920533736546835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,float16,31,0.16145599881807962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,fp8,31,0.159360001484553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,float16,63,0.16238933801651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,fp8,63,0.16074666380882263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,float16,127,0.19851734240849814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,fp8,127,0.19696533679962158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,float16,255,0.31012799342473346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,fp8,255,0.30829866727193195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,float16,511,0.5424586534500122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,fp8,511,0.5293066501617432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,float16,1023,0.9901493390401205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,fp8,1023,0.9852480093638102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,1,0.03409600009520849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,3,0.03427733232577642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,7,0.034186666210492454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,15,0.03417599946260452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,31,0.03364799916744232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,63,0.03356799980004629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,127,0.03333866596221924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,255,0.06514666477839152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,511,0.08223466575145721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,1023,0.10853333274523418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,1,0.046384001771608986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,1,0.017621333400408428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,3,0.04665599763393402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,3,0.017711999515692394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,7,0.04641599953174591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,7,0.017551999539136887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,15,0.04636266827583313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,15,0.01758933315674464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,31,0.046256000796953835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,31,0.017525333911180496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,63,0.04595733185609182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,63,0.01746133342385292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,127,0.04781866570313772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,127,0.02176533391078313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,255,0.056645333766937256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,255,0.031354665756225586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,511,0.07868266602357228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,511,0.04507733384768168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,1023,0.12335999806722005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,1023,0.06791999936103821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,1,0.22235733270645142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,3,0.23378666241963705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,fp8,3,0.23812266190846762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,7,0.24523733059565225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,fp8,7,0.24978667497634888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,fp8,1,0.22824533780415854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,15,0.3163413405418396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,fp8,15,0.31489600737889606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,31,0.3165493408838908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,fp8,31,0.31483733654022217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,63,0.31884799400965375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,fp8,63,0.31799999872843426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,127,0.38948265711466473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,fp8,127,0.3894186814626058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,255,0.6162240107854208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,fp8,255,0.6120906670888265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,128,0,1,float16,float16,1,0.22436267137527466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,128,0,1,float16,fp8,1,0.22986666361490884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,128,0,1,float16,float16,3,0.23396267493565878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,128,0,1,float16,fp8,3,0.23811733722686768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,128,0,1,float16,float16,7,0.2454026738802592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,128,0,1,float16,fp8,7,0.24972800413767496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,128,0,1,float16,float16,15,0.31640533606211346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,128,0,1,float16,fp8,15,0.31513067086537677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,128,0,1,float16,float16,31,0.3165973424911499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,128,0,1,float16,fp8,31,0.3149973352750142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,128,0,1,float16,float16,63,0.31915199756622314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,128,0,1,float16,fp8,63,0.3181333343187968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,128,0,1,float16,float16,127,0.39690132935841876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,128,0,1,float16,fp8,127,0.3898933331171672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,128,0,1,float16,float16,255,0.6253759860992432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,1,0.061749334136645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,128,0,1,float16,fp8,255,0.6196959813435873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,3,0.06195733447869619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,7,0.0610453337430954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,15,0.06069866816202799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,31,0.061205332477887474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,63,0.06202666461467743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,127,0.06374933322270711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,255,0.07283733288447063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,1,0.08441600203514099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,fp8,1,0.031114667654037476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,3,0.08463467160860698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,fp8,3,0.03166933357715607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,7,0.08454933762550354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,fp8,7,0.03169066707293192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,15,0.08494399984677632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,fp8,15,0.03161066770553589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,31,0.08542933066685994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,fp8,31,0.03149333347876867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,63,0.08604799707730611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,fp8,63,0.03143466760714849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,127,0.08754133184750874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,fp8,127,0.04947733382383982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,fp8,255,0.059792002042134605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,1,0.4394773244857788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,fp8,1,0.4507146676381429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,3,0.46454934279123944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,fp8,3,0.47139732042948407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,7,0.48598400751749676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,fp8,7,0.4947413206100464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,15,0.6278186639149984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,fp8,15,0.6249759991963705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,31,0.6291306813557943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,255,0.10069866975148518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,fp8,31,0.6250453392664591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,fp8,63,0.6323413451512655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,63,0.6366186539332072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,127,0.7897866566975912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,fp8,127,0.7747093041737875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,128,0,1,float16,float16,1,0.44284268220265705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,128,0,1,float16,fp8,1,0.45045868555704754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,128,0,1,float16,float16,3,0.4625493288040161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,128,0,1,float16,fp8,3,0.4678826729456584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,128,0,1,float16,float16,7,0.48747201760609943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,128,0,1,float16,fp8,7,0.4920053482055664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,128,0,1,float16,float16,15,0.6296480099360148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,128,0,1,float16,fp8,15,0.6217386722564697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,128,0,1,float16,float16,31,0.6368213494618734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,128,0,1,float16,fp8,31,0.6235733429590861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,128,0,1,float16,float16,63,0.6518773237864176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,128,0,1,float16,fp8,63,0.6356053352355957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,128,0,1,float16,float16,127,0.7996106942494711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,128,0,1,float16,fp8,127,0.7904906272888184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,1,0.11489599943161011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,3,0.11572266618410747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,7,0.11500799655914307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,15,0.1151626706123352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,31,0.11531733473141988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,63,0.11596266428629558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,127,0.11591999729474385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,1,0.1574133336544037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,fp8,1,0.07276266813278198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,fp8,3,0.07294400036334991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,3,0.15854400396347046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,15,0.16009599963823953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,fp8,15,0.07310933371384938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,31,0.16059733430544534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,fp8,31,0.07276266813278198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,63,0.15959999958674112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,fp8,63,0.07262399792671204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,127,0.16044800480206808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,fp8,127,0.08719467123349507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,fp8,7,0.07300800085067749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,7,0.15971199671427408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,1,0.012549333274364471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,3,0.012826666235923767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,7,0.012725333372751871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,1,0.012645332763592402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,7,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,15,0.014677333335081736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,3,0.01302933320403099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,15,0.01471466695268949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,31,0.014837333311637243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,63,0.014890667051076889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,31,0.014826666563749313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,63,0.014975999792416891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,127,0.017370666066805523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,127,0.017397332936525345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,255,0.02498133232196172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,255,0.024842667082945507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,511,0.039808000127474465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,511,0.04114133367935816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,1023,0.06980800131956737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,1023,0.07217599948247273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,2047,0.1301759978135427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,2047,0.1338879962762197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,4095,0.24876266717910767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,4095,0.2590293288230896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,8191,0.5000746647516886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,8191,0.5076533158620199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,1,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,fp8,1,0.013072000195582708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,3,0.013194666554530462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,16383,1.335861365000407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,fp8,3,0.01339200014869372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,16383,1.1973706881205242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,7,0.012768000364303589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,fp8,7,0.01320533330241839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,fp8,15,0.01470400020480156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,15,0.014965333044528961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,31,0.015029333531856537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,fp8,31,0.014730667074521383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,63,0.01524266724785169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,fp8,63,0.015205333630243937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,127,0.01741333305835724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,fp8,127,0.017674667139848072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,255,0.02514133354028066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,fp8,255,0.02508266766866048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,511,0.04005866746107737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,fp8,511,0.04116799930731455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,1023,0.07043200234572093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,fp8,1023,0.07229866584142049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,2047,0.13154666622479758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,fp8,2047,0.13446399569511414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,4095,0.2521066665649414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,fp8,4095,0.257152001063029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,8191,0.5337813297907511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,fp8,8191,0.5066879987716675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,16383,1.3517813682556152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,fp8,16383,1.453994592030843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,1,0.01267733300725619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,3,0.013072000195582708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,7,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,15,0.014629332969586054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,31,0.014677333335081736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,63,0.014970666418472925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,127,0.017466666797796886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,255,0.014688000082969666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,511,0.015290666371583939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,1023,0.022848000129063923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,2047,0.034117333590984344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,4095,0.04980266590913137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,8191,0.07408533493677776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,16383,0.10974400242169698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,1,0.010490667074918747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,1,0.006837333242098491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,3,0.010314666976531347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,3,0.006874666859706243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,7,0.0106133334338665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,7,0.006757333253820737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,15,0.010634666929642359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,15,0.00696000022192796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,31,0.010634666929642359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,31,0.007002666592597961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,63,0.010608000059922537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,63,0.006927999978264173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,127,0.010533332824707031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,127,0.007536000261704127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,255,0.013669333110253016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,255,0.009162666896979014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,511,0.01964266722400983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,511,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,1023,0.02863999952872594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,1023,0.018005333840847015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,2047,0.04397333165009817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,2047,0.022384000321229298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,4095,0.06831466654936473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,4095,0.036346666514873505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,8191,0.08685333530108134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,8191,0.04884799818197886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,16383,0.12993599971135458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,16383,0.07344000041484833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,float16,1,0.9284213383992513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,fp8,1,0.9486026763916016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,float16,3,0.9523786703745524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,fp8,3,0.9706772963205973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,float16,7,1.0003306865692139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,fp8,7,1.0177866617838542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,float16,15,1.2541653315226238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,fp8,15,1.258565346399943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,float16,31,1.2780959606170654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,fp8,31,1.2684426307678223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,float16,63,1.289749304453532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,fp8,63,1.281557321548462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,128,0,1,float16,float16,1,0.9463573296864828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,128,0,1,float16,fp8,1,0.9671839872996012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,128,0,1,float16,float16,3,0.9679839611053467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,128,0,1,float16,fp8,3,0.9854666392008463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,128,0,1,float16,float16,7,1.0136746565500896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,128,0,1,float16,fp8,7,1.0313173135121663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,128,0,1,float16,float16,15,1.274938662846883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,128,0,1,float16,fp8,15,1.2702666918436687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,128,0,1,float16,float16,31,1.2846986452738445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,128,0,1,float16,fp8,31,1.2751413186391194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,128,0,1,float16,float16,63,1.2976853052775066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,128,0,1,float16,fp8,63,1.2907466888427734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,float16,1,0.2243573268254598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,float16,3,0.22312533855438232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,float16,7,0.22175999482472739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,float16,15,0.22181866566340128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,float16,31,0.22286399205525717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,float16,63,0.22127467393875122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,float16,1,0.3131200075149536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,fp8,1,0.13635200262069702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,float16,3,0.31356267134348553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,fp8,3,0.13589866956075033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,float16,7,0.31413867076237995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,fp8,7,0.13620799779891968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,float16,15,0.3144266605377197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,fp8,15,0.13612266381581625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,fp8,31,0.13572266697883606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,float16,63,0.31099732716878253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,fp8,63,0.1354986627896627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,float16,1,1.83732271194458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,fp8,1,1.8764373461405437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,float16,31,0.3116640051205953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,float16,3,1.8842666943868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,float16,7,1.9807252883911133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,fp8,3,1.9223413467407227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,fp8,7,2.0151146252950034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,float16,15,2.5200907389322915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,fp8,15,2.510437329610189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,float16,31,2.5316640535990396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,fp8,31,2.5126026471455893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,2,128,0,1,float16,float16,1,1.872037410736084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,2,128,0,1,float16,fp8,1,1.9150667190551758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,2,128,0,1,float16,float16,3,1.917093276977539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,2,128,0,1,float16,fp8,3,1.9498400688171387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,2,128,0,1,float16,float16,7,2.0052693684895835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,2,128,0,1,float16,fp8,7,2.040613333384196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,2,128,0,1,float16,float16,15,2.5244266192118325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,2,128,0,1,float16,fp8,15,2.5155359903971353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,2,128,0,1,float16,float16,31,2.5452213287353516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,2,128,0,1,float16,fp8,31,2.526693344116211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,float16,1,0.4346986611684163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,float16,3,0.4341226816177368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,float16,7,0.4368853171666463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,float16,15,0.43530134359995526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,float16,31,0.4322346846262614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,float16,1,0.6164213418960571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,fp8,1,0.26128000020980835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,float16,3,0.6188053290049235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,fp8,3,0.2604159911473592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,float16,15,0.6164426803588867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,fp8,15,0.2615306576093038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,float16,7,0.6206399997075399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,float16,31,0.6143360137939453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,fp8,7,0.261898676554362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,1,0.01814933369557063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,1,0.018506667266289394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,fp8,31,0.26069867610931396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,3,0.01860800012946129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,3,0.019189332922299702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,7,0.01964266722400983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,15,0.02442666639884313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,15,0.02421333392461141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,7,0.01958400011062622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,31,0.024559999505678814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,31,0.02479466547568639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,63,0.024800000091393787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,63,0.024735999604066212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,127,0.029653333127498627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,127,0.02943466603755951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,255,0.04399999976158142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,255,0.04437866806983948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,511,0.07372266550858815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,511,0.07423999905586243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,1023,0.13194132844607034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,1023,0.1335040032863617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,2047,0.24730134010314941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,2047,0.2516746719678243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,4095,0.480074683825175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,4095,0.4869920015335083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,float16,1,0.01809599995613098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,fp8,1,0.018496000518401463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,8191,0.9643627007802328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,8191,0.9739573001861572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,float16,3,0.018810667097568512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,fp8,3,0.019365333020687103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,float16,7,0.019589333484570186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,float16,15,0.0244159996509552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,fp8,7,0.01966933285196622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,fp8,15,0.02443733314673106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,float16,31,0.024570666253566742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,fp8,31,0.024373332659403484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,float16,63,0.024677333732446034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,fp8,63,0.02479466547568639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,float16,127,0.029365333418051403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,fp8,127,0.029498666524887085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,float16,255,0.04428799947102865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,fp8,255,0.04420266548792521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,float16,511,0.07341866691907246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,fp8,511,0.07425599793593089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,float16,1023,0.13145599762598673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,fp8,1023,0.13355200489362082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,float16,2047,0.2490346630414327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,fp8,2047,0.2511039972305298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,float16,4095,0.49181334177652997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,fp8,4095,0.4861493508021037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,float16,8191,0.9798080126444498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,fp8,8191,0.9852000077565511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,1,0.01232533281048139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,3,0.012565333396196365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,7,0.012367999802033106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,15,0.01221866657336553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,31,0.01232533281048139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,63,0.012293333808581034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,127,0.01219733307758967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,255,0.015941333025693893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,511,0.02317333221435547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,1023,0.03329599897066752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,2047,0.05348266661167145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,4095,0.0823413332303365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,8191,0.10709866881370544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,1,0.010709332923094431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,1,0.007034666836261749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,3,0.01051733394463857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,3,0.007018666714429855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,7,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,7,0.0069440001000960665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,15,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,15,0.0068693334857622785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,31,0.010682666053374609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,31,0.006911999856432279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,63,0.0069440001000960665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,127,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,127,0.007743999982873599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,255,0.021381333470344543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,255,0.010351999973257383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,511,0.0286613330245018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,63,0.010437333335479101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,511,0.017344000438849132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,1023,0.046442667643229164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,1023,0.02588266630967458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,2047,0.06446933249632518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,2047,0.0340639998515447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,4095,0.08406399687131245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,4095,0.0492799977461497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,8191,0.1265546679496765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,8191,0.07517333328723907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,1,0.023717333873112995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,1,0.023792001108328503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,1,0.024069334069887798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,3,0.02365333338578542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,3,0.02388266722361247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,3,0.023823998868465424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,3,0.024106666445732117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,7,0.02463999887307485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,7,0.02481599897146225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,7,0.02478933334350586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,7,0.025050667424996693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,15,0.025429333249727886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,15,0.025701334079106648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,15,0.025381334125995636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,15,0.025813333690166473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,31,0.03146133323510488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,31,0.03188266605138779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,31,0.03147733211517334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,31,0.03194133440653483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,63,0.031685332457224526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,63,0.03204799940188726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,63,0.03182933231194814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,63,0.03178666780392329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,1,0.023503998915354412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,127,0.03180266668399175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,127,0.03196800003449122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,127,0.03179199993610382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,255,0.03197333216667175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,127,0.03187733391920725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,255,0.03822933385769526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,255,0.03193599979082743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,255,0.037717332442601524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,511,0.03218133250872294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,511,0.05721066892147064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,511,0.031914666295051575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,511,0.057850668827692665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,1023,0.03219199925661087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,1023,0.09510933359464009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,1023,0.032127998769283295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,1023,0.09806933005650838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,2047,0.03203733265399933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,2047,0.17113600174585977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,2047,0.03221333275238673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,4095,0.03194133440653483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,2047,0.17463467518488565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,4095,0.032032000521818794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,4095,0.3259359995524089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,4095,0.3303733269373576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,8191,0.03203733265399933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,8191,0.03182933231194814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,8191,0.6547520160675049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,8191,0.643120010693868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,1,0.011413333316644033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,1,0.01126933346192042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,3,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,3,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,1,0.00816000004609426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,3,0.008229333286484083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,1,0.008223999912540117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,7,0.011306667079528173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,7,0.008629333227872849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,7,0.008629333227872849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,15,0.011584000041087469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,7,0.011472000430027643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,15,0.011535999675591787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,15,0.008725333337982496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,15,0.008522666369875273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,31,0.011440000186363855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,31,0.011578666667143503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,3,0.008469333251317343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,31,0.008549333239595095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,31,0.00860799973209699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,63,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,63,0.008757333581646284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,63,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,63,0.008357333640257517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,127,0.01145600030819575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,127,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,127,0.009269333134094873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,127,0.00955200009047985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,255,0.013568000247081121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,255,0.013546666751305262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,255,0.009701333319147428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,255,0.010426666587591171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,511,0.019205333044131596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,511,0.019968000551064808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,511,0.010202666744589806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,511,0.012037333101034164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,1023,0.025013332565625507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,1023,0.02903999884923299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,1023,0.010570666442314783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,1023,0.02037866661945979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,2047,0.031957333286603294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,2047,0.04237333436806997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,2047,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,2047,0.024671999116738636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,4095,0.040037333965301514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,4095,0.06411199768384297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,4095,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,4095,0.03868266691764196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,8191,0.08193600177764893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,8191,0.039887999494870506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,8191,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,8191,0.05629333357016245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,1,0.010522666076819101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,1,0.01032533310353756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,3,0.01032533310353756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,3,0.010415999839703241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,1,0.006789333497484525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,3,0.006671999891599019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,1,0.0069759997228781385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,7,0.010378666842977205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,3,0.00707733320693175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,7,0.006858666737874349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,7,0.0069919998447100324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,15,0.010346666599313417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,15,0.010480000327030817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,15,0.006837333242098491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,15,0.006965333595871925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,31,0.01032533310353756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,31,0.010591999938090643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,31,0.0069333333522081375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,31,0.007034666836261749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,63,0.010239999741315842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,63,0.010421333213647207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,63,0.006927999978264173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,63,0.0069866664707660675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,127,0.010213333492477735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,127,0.010485333700974783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,127,0.007274666801095009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,127,0.007280000175038974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,255,0.018298666924238205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,255,0.01826133330663045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,255,0.008714666590094566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,255,0.009493333597977957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,511,0.023002666731675465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,511,0.02521066615978877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,511,0.009621333330869675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,511,0.011653333902359009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,1023,0.03339199970165888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,1023,0.0402399996916453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,1023,0.009712000067035357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,1023,0.020154666155576706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,2047,0.03563733398914337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,2047,0.05308799942334493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,2047,0.009610666582981745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,7,0.010239999741315842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,2047,0.024613333245118458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,4095,0.0356480007370313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,4095,0.06681066751480103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,4095,0.009706666693091393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,4095,0.03495999922355016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,8191,0.03551466763019562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,8191,0.09034132957458496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,8191,0.009839999799927076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,8191,0.055488000313440956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,1,0.006480000292261441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,1,0.006511999915043513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,1,0.014607999473810196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,7,0.014405333747466406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,1,0.015018666783968607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,7,0.014432000617186228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,3,0.01565333331624667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,15,0.014570667097965876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,15,0.014544000228246054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,15,0.006319999694824219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,15,0.006351999938488007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,31,0.014607999473810196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,31,0.014544000228246054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,31,0.0063146669417619705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,3,0.006239999706546466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,31,0.006261333202322324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,63,0.01441066712141037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,63,0.006778666749596596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,63,0.006224000205596288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,127,0.014256000518798828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,3,0.014565333724021912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,3,0.006208000083764394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,127,0.014287999520699183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,127,0.0069440001000960665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,127,0.0068693334857622785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,255,0.026250667870044708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,255,0.02682666728893916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,255,0.008346666892369589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,255,0.009535999968647957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,511,0.03628266602754593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,511,0.03992533435424169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,511,0.01003200002014637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,511,0.016800000021855038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,1023,0.03619733452796936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,7,0.006389333556095759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,7,0.006458666796485583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,1023,0.049866666396458946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,1023,0.00997866690158844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,1023,0.020117333779732387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,2047,0.038922667503356934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,2047,0.06694933275381725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,2047,0.009888000165422758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,2047,0.02844800055027008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,4095,0.03912533322970072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,4095,0.08929066856702168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,4095,0.010048000141978264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,4095,0.04574933151404063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,8191,0.03875733415285746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,8191,0.13335466384887695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,8191,0.009919999788204828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,1,0.00766933336853981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,8191,0.0780266672372818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,1,0.00786666696270307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,1,0.00895999992887179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,3,0.007733333234985669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,3,0.007813333223263422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,3,0.008954666554927826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,3,0.008949333180983862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,7,0.007770666852593422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,7,0.007765333478649457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,7,0.008912000184257826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,7,0.00892800030608972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,15,0.007791999727487564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,15,0.007717333113153775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,1,0.008992000172535578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,15,0.008943999807039896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,15,0.00903466654320558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,31,0.00795199970404307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,31,0.008058666562040647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,31,0.009119999905427298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,31,0.00915733352303505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,63,0.008613333106040955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,63,0.008736000085870424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,63,0.00979200005531311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,63,0.009882666791478792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,127,0.008879999940594038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,127,0.008965333302815756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,127,0.01128000020980835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,127,0.011402666568756104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,255,0.008885333314538002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,255,0.008943999807039896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,255,0.011391999820868174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,255,0.011445333560307821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,511,0.008938666433095932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,511,0.0100853331387043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,511,0.011328000575304031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,1023,0.014165333161751429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,511,0.011477333803971609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,1023,0.015935999651749928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,1023,0.013770667215188345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,1023,0.01591466615597407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,2047,0.02049066623051961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,2047,0.027552001178264618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,2047,0.02048533285657565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,2047,0.02717333287000656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,4095,0.0206133338312308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,4095,0.03495466709136963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,4095,0.020975999534130096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,4095,0.03443733354409536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,8191,0.04794133206208547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,8191,0.020629333953062694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,8191,0.020479999482631683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,8191,0.047151997685432434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,63,0.014138666292031607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,16383,0.07506133119265239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,16383,0.020645332833131153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,16383,0.02038399999340375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,16383,0.07381866872310638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,1,0.007882666463653246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,32767,0.12794666488965353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,32767,0.020666666328907013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,1,0.00789866658548514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,32767,0.12588799993197122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,1,0.00785600021481514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,3,0.00789866658548514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,32767,0.020207999895016353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,3,0.007690666864315669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,3,0.007818666597207388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,7,0.00786666696270307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,7,0.00772266648709774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,7,0.007818666597207388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,15,0.007871999715765318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,15,0.00772266648709774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,15,0.007802666475375493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,15,0.008362666393319765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,31,0.007887999837597212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,31,0.008037333066264788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,31,0.008336000144481659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,31,0.008127999802430471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,63,0.008709333216150602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,63,0.00854399986565113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,63,0.007861333588759104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,63,0.008117333054542542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,127,0.008799999952316284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,127,0.0086666668454806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,127,0.008661333471536636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,127,0.009114666531483332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,255,0.008842666943868002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,255,0.008816000074148178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,255,0.009610666582981745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,255,0.010458666831254959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,511,0.008826666822036108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,511,0.009818666925032934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,511,0.009648000200589498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,511,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,1023,0.014010666559139887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,1023,0.01598400001724561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,1023,0.009818666925032934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,1023,0.012058666596810022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,2047,0.020453333854675293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,2047,0.027503999571005504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,2047,0.009674666449427605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,2047,0.013167999684810638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,4095,0.014335999886194864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,4095,0.014378666877746582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,4095,0.009679999823371569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,4095,0.014906667172908783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,8191,0.01479999969402949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,8191,0.01748266691962878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,8191,0.009248000259200731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,8191,0.01724799970785777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,16383,0.015989333391189575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,16383,0.018415999909241993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,16383,0.009509333098928133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,16383,0.020474666108687718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,32767,0.01762666677435239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,32767,0.02223466585079829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,32767,0.009413333609700203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,32767,0.026202666262785595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,1,0.007840000092983246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,1,0.007791999727487564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,1,0.006586666529377301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,1,0.006495999793211619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,3,0.007887999837597212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,3,0.007802666475375493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,3,0.006831999868154526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,3,0.006490666419267654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,1,0.008117333054542542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,7,0.007887999837597212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,7,0.007813333223263422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,3,0.007861333588759104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,7,0.006805333619316419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,15,0.007760000104705493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,7,0.008010666817426682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,15,0.006464000170429547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,15,0.006831999868154526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,31,0.007882666463653246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,31,0.006789333497484525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,31,0.006538666784763336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,63,0.00873066671192646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,63,0.008650666723648706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,63,0.006805333619316419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,63,0.006693333387374878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,127,0.00879466657837232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,127,0.008789333204428354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,127,0.0074879998962084455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,127,0.007061333085099856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,255,0.008805333326260248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,255,0.008879999940594038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,255,0.008522666369875273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,255,0.007829333345095316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,511,0.008821333448092142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,511,0.009743999689817429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,511,0.007957333077987036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,511,0.009216000015536943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,1023,0.013893333574136099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,1023,0.015802666544914246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,1023,0.008277333031098047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,1023,0.009786666681369146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,2047,0.01209066684047381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,2047,0.012357333054145178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,2047,0.007786666974425316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,2047,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,4095,0.012576000144084295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,4095,0.013050666699806849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,4095,0.007791999727487564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,4095,0.012256000190973282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,8191,0.013093333691358566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,8191,0.015552000453074774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,8191,0.007978666573762894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,8191,0.014629332969586054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,16383,0.017018667111794155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,16383,0.013823999712864557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,16383,0.007802666475375493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,16383,0.017621333400408428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,32767,0.026309333741664886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,32767,0.01741333305835724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,32767,0.007978666573762894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,1,0.007658666620651881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,1,0.007701333612203598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,32767,0.024304000039895374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,1,0.005850666513045629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,1,0.005701333284378052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,3,0.007882666463653246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,3,0.007786666974425316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,3,0.0058559998869895935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,3,0.005728000154097875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,7,0.007791999727487564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,7,0.007743999982873599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,7,0.005775999898711841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,7,0.006010666489601135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,15,0.00784533346692721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,15,0.007765333478649457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,15,0.0058613332609335584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,15,0.005978666866819064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,31,0.007946666950980822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,31,0.008037333066264788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,31,0.005797333394487699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,31,0.005989333614706993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,7,0.006800000245372455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,15,0.007850666840871176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,31,0.008069333309928576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,63,0.008586666857202848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,63,0.008714666590094566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,63,0.006042666733264923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,63,0.006618666773041089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,127,0.008842666943868002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,127,0.00877333308259646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,127,0.0063146669417619705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,127,0.006346666564544042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,255,0.008901333436369896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,255,0.0069919998447100324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,255,0.007642666498819987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,511,0.009770666559537252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,511,0.009136000027259191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,511,0.007967999825874964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,511,0.007205333560705185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,1023,0.011786667009194693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,1023,0.0122079998254776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,1023,0.008367999767263731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,1023,0.007194666812817256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,2047,0.01210133358836174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,2047,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,2047,0.00916800027092298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,2047,0.007184000064929326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,4095,0.012304000556468964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,4095,0.007247999931375186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,4095,0.0136266661187013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,4095,0.010346666599313417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,8191,0.015290666371583939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,8191,0.020138667275508244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,8191,0.0074560002734263735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,8191,0.013658666362365087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,16383,0.01642666632930438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,16383,0.02348800003528595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,16383,0.007402666533986728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,16383,0.016410666207472484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,1,0.007914666707317034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,32767,0.02006400004029274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,32767,0.007285333548982938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,1,0.007914666707317034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,32767,0.0356480007370313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,32767,0.022800001005331676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,1,0.008965333302815756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,1,0.009018666421373686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,3,0.007920000081261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,3,0.007903999959429106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,3,0.00914666677514712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,3,0.009008000294367472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,7,0.007871999715765318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,7,0.009093333035707474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,7,0.007946666950980822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,7,0.00915733352303505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,15,0.007797333101431529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,15,0.009109333157539368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,15,0.007834666719039282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,15,0.009098666409651438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,31,0.008298666526873907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,31,0.008122666428486506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,31,0.009349333122372627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,31,0.009130666653315226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,63,0.008896000062425932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,63,0.009029333169261614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,63,0.009861333295702934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,63,0.009925333162148794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,127,0.008890666688481966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,127,0.01157333329319954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,127,0.009061333412925402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,255,0.009002666920423508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,127,0.011567999919255575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,255,0.009029333169261614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,255,0.008874666566650072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,255,0.011509332805871964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,255,0.011589333415031433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,511,0.008885333314538002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,511,0.010053333515922228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,511,0.011509332805871964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,511,0.011898666620254517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,1023,0.021957332889238994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,1023,0.02611200014750163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,1023,0.02179199953873952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,2047,0.030224000414212544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,1023,0.026015999416510265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,2047,0.02214933435122172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,2047,0.021898667017618816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,2047,0.029994666576385498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,4095,0.02187199890613556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,4095,0.04067199925581614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,4095,0.021957332889238994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,4095,0.040250666439533234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,8191,0.02170666555563609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,8191,0.06005866825580597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,8191,0.02201066662867864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,8191,0.06046933432420095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,16383,0.021903999149799347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,16383,0.09983467062314351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,16383,0.021882665654023487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,16383,0.10013866424560547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,32767,0.17938133080800375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,32767,0.021840001145998638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,32767,0.022064000368118286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,32767,0.17811199029286703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,1,0.007871999715765318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,1,0.0075626665105422335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,1,0.00786666696270307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,3,0.00785600021481514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,3,0.007797333101431529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,3,0.007605333502093951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,3,0.008309333274761835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,7,0.00784533346692721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,7,0.007823999971151352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,7,0.00821333316465219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,7,0.008341333518425623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,15,0.007871999715765318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,15,0.007802666475375493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,15,0.008282666405042013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,15,0.008277333031098047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,31,0.008181333541870117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,31,0.007983999947706858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,31,0.008256000156203905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,31,0.008298666526873907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,63,0.00879466657837232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,63,0.008709333216150602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,63,0.008197333042820295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,127,0.009082666908701261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,63,0.008197333042820295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,127,0.008933333059151968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,127,0.009189333145817121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,255,0.008890666688481966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,255,0.00892800030608972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,255,0.009658666948477427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,255,0.010666667173306147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,511,0.008901333436369896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,511,0.009984000275532404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,511,0.009717333440979322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,511,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,1023,0.02181866765022278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,1023,0.026101333399613697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,1023,0.009626666704813639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,1023,0.01192533348997434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,2047,0.014362666755914688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,2047,0.014538666854302088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,2047,0.009370666618148485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,2047,0.013232000172138214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,4095,0.014688000082969666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,4095,0.015279999623696009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,4095,0.009232000137368837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,4095,0.01443733274936676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,8191,0.015360000232855478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,8191,0.018474667022625606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,8191,0.009237333511312803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,8191,0.017312000195185345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,16383,0.016229332735141117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,16383,0.020090666910012562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,16383,0.009258666386206945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,16383,0.020928000410397846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,32767,0.020453333854675293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,32767,0.031658666829268135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,32767,0.00943999985853831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,1,0.007727999861041705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,1,0.00795199970404307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,1,0.0064213331788778305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,32767,0.029450667401154835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,1,0.006666666517655055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,3,0.007807999849319458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,3,0.00786666696270307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,3,0.00660800002515316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,3,0.006405333057045937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,7,0.007877333089709282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,7,0.007946666950980822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,7,0.006394666930039723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,7,0.006864000111818314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,15,0.007978666573762894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,15,0.007781333600481351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,15,0.006842666616042455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,15,0.006890666360656421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,31,0.008026666939258575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,31,0.008165333420038223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,1,0.008026666939258575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,31,0.00684799998998642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,31,0.006810666372378667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,63,0.008912000184257826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,63,0.006709333509206772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,127,0.009119999905427298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,127,0.00884799969693025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,127,0.007594666754206021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,127,0.007205333560705185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,255,0.009114666531483332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,255,0.008954666554927826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,255,0.008330666770537695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,255,0.008816000074148178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,511,0.009173333023985228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,511,0.009941333283980688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,511,0.008266666904091835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,511,0.009434666484594345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,1023,0.01211200033624967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,1023,0.012063999970753988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,1023,0.008176000167926153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,1023,0.009583999713261923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,2047,0.012928000340859095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,2047,0.013429333766301474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,2047,0.008005333443482717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,2047,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,4095,0.012986666212479273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,4095,0.01394133393963178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,4095,0.007962666451931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,4095,0.012309333930412928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,127,0.009072000160813332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,8191,0.016208000481128693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,8191,0.020874666670958202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,8191,0.016496000190575916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,16383,0.01728533332546552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,16383,0.02441066751877467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,16383,0.008447999755541483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,16383,0.020053333292404812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,32767,0.02089600016673406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,32767,0.036320000886917114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,32767,0.008303999900817871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,1,0.007813333223263422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,32767,0.02882666637500127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,1,0.007871999715765318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,1,0.005946666623155276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,1,0.0058559998869895935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,3,0.007989333321650824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,3,0.007834666719039282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,3,0.006026666611433029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,3,0.005829333638151486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,7,0.00797333319981893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,7,0.007887999837597212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,7,0.0058453331391016645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,7,0.006053333481152852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,15,0.007877333089709282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,15,0.007861333588759104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,15,0.006010666489601135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,15,0.005957333371043205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,31,0.00797333319981893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,31,0.008245333408315977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,31,0.005968000118931134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,31,0.005904000252485275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,63,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,63,0.008832000195980072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,63,0.005877333382765452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,63,0.006010666489601135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,127,0.008976000050703684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,127,0.00890666681031386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,127,0.0064266665528217954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,127,0.006351999938488007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,255,0.009103999783595404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,63,0.00891733355820179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,255,0.007184000064929326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,255,0.008879999940594038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,255,0.007791999727487564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,511,0.011658667276302973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,511,0.011637333780527115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,511,0.0069973332186539965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,511,0.008112000301480293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,1023,0.011648000528415045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,1023,0.011887999872366587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,1023,0.00843733362853527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,1023,0.007151999821265538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,2047,0.015311999867359797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,2047,0.017344000438849132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,2047,0.007344000041484833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,2047,0.009919999788204828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,4095,0.016037333756685257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,63,0.00666133314371109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,4095,0.019167999426523846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,4095,0.00744000015159448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,4095,0.01139733319481214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,8191,0.019071999937295914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,8191,0.00730666642387708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,8191,0.015386667102575302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,16383,0.022272000710169475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,16383,0.03788800040880839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,16383,0.0074879998962084455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,16383,0.021738665799299877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,32767,0.02777066578467687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,32767,0.06025066475073496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,32767,0.007525333513816197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,32767,0.03565333286921183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,1,0.04200000067551931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,8191,0.027402666707833607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,1,0.04204800228277842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,1,0.04234133164087931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,1,0.04212800165017446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,3,0.04242666562398275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,3,0.04204266766707102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,3,0.042378668983777366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,3,0.04214933514595032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,7,0.0439573327700297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,7,0.04367466767628988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,7,0.04411733150482178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,7,0.04371733466784159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,8191,0.007962666451931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,15,0.04548266530036926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,15,0.045269335309664406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,31,0.05665066838264465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,15,0.04563733438650767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,15,0.04540266593297323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,31,0.057087997595469155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,63,0.05766933163007101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,31,0.05726933479309082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,31,0.05705066521962484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,63,0.057376002271970115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,63,0.058101331194241844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,63,0.05811200042565664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,127,0.058304001887639366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,127,0.058693334460258484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,127,0.05811200042565664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,127,0.05821866790453593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,255,0.058575997749964394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,255,0.07022400200366974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,255,0.05852800110975901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,255,0.0701333334048589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,511,0.0584799995024999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,511,0.10851732889811198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,511,0.05829333265622457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,511,0.10969066619873047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,1023,0.05845333139101664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,1023,0.18289067347844443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,1023,0.05858133236567179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,1023,0.19128000736236572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,2047,0.058821335434913635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,2047,0.058362667759259544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,2047,0.3326293428738912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,2047,0.3407626549402873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,4095,0.05860800047715505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,4095,0.05816000203291575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,1,0.011941333611806234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,1,0.011999999483426413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,1,0.008053333188096682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,1,0.00820266641676426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,4095,0.6428426504135132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,3,0.012122667084137598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,3,0.011855999628702799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,3,0.008117333054542542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,4095,0.6436053514480591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,7,0.012026666353146235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,7,0.011786667009194693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,7,0.008197333042820295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,7,0.008063999935984612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,15,0.012074666718641916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,15,0.011882666498422623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,15,0.008218666538596153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,15,0.008053333188096682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,31,0.011893333246310553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,31,0.011765333513418833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,31,0.00820266641676426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,31,0.008453333129485449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,63,0.011770666887362799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,63,0.011727999895811081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,63,0.008186666915814081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,63,0.008442666381597519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,127,0.01173866664369901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,127,0.011674666156371435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,127,0.009061333412925402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,127,0.00890666681031386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,255,0.021546666820844013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,255,0.021317332983016968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,255,0.01020800011853377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,255,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,511,0.02700799951950709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,511,0.02962133288383484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,511,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,1023,0.03893866638342539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,511,0.019093333433071773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,1023,0.047509332497914634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,1023,0.028170667588710785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,1023,0.011813333878914515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,2047,0.04138666639725367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,2047,0.063701331615448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,2047,0.011434666812419891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,2047,0.03645866612593333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,4095,0.04167999823888143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,4095,0.08102400104204814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,4095,0.011653333902359009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,1,0.014874666929244995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,4095,0.05630399783452352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,1,0.007141333073377609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,1,0.007231999809543292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,1,0.015413332730531693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,3,0.01488000030318896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,3,0.014938666174809137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,3,0.007285333548982938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,3,0.007221333061655362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,7,0.014970666418472925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,7,0.014922666052977243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,7,0.007082666580875714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,7,0.007216000308593114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,15,0.014890667051076889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,15,0.015087999403476715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,15,0.007189333438873291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,15,0.007274666801095009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,31,0.01498666654030482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,31,0.014997333288192749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,31,0.007034666836261749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,31,0.007269333427151044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,63,0.014842666685581207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,63,0.014789332946141561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,63,0.006965333595871925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,63,0.007151999821265538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,127,0.014837333311637243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,127,0.008021333565314611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,127,0.01482133318980535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,127,0.008101333553592363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,255,0.026895999908447266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,255,0.009701333319147428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,255,0.027098665634791057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,255,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,511,0.036474667489528656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,511,0.04112533231576284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,511,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,511,0.019146667172511418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,1023,0.036415999134381614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,1023,0.05067199965318044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,1023,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,1023,0.02372266600529353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,2047,0.03893866638342539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,2047,0.06795733173688252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,2047,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,2047,0.03453866640726725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,4095,0.03870933254559835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,4095,0.09057066837946574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,4095,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,1,0.023232000569502514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,4095,0.0555626650651296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,1,0.023232000569502514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,1,0.006981333096822103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,1,0.006927999978264173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,3,0.008101333553592363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,3,0.023728000621000927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,3,0.023247999449570973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,3,0.007087999954819679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,7,0.023306667804718018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,7,0.023578666150569916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,3,0.006949333474040031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,7,0.00702400008837382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,15,0.023584000766277313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,7,0.007135999699433644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,15,0.023685333629449207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,15,0.006954666847983996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,31,0.023605334262053173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,31,0.0235359991590182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,15,0.007109333450595538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,31,0.007231999809543292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,31,0.0069440001000960665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,63,0.02327466756105423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,63,0.023573334018389385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,63,0.006981333096822103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,63,0.007162666569153468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,127,0.02316266546646754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,127,0.023685333629449207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,127,0.008362666393319765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,127,0.008240000034372011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,255,0.043466667334238686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,255,0.01607999950647354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,255,0.04355733096599579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,255,0.013893333574136099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,511,0.013733333597580591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,511,0.04363200068473816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,511,0.05332266787687937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,511,0.02027733375628789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,1023,0.06611733138561249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,1023,0.04394666850566864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,1023,0.013839999834696451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,1023,0.02882666637500127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,2047,0.04647466540336609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,2047,0.013807999591032663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,2047,0.04650133351484934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,4095,0.0462666650613149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,2047,0.09273599584897359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,4095,0.1361066699028015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,4095,0.07844266792138417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,1,0.008709333216150602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,4095,0.013845333208640417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,1,0.00855466661353906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,1,0.008709333216150602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,1,0.008506666868925095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,3,0.00871999996403853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,3,0.00860799973209699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,3,0.008570666735370954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,7,0.008912000184257826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,7,0.008869333192706108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,3,0.008746666833758354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,7,0.00879466657837232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,7,0.008639999975760778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,15,0.008912000184257826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,15,0.008901333436369896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,15,0.008810666700204214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,15,0.008799999952316284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,31,0.009392000113924345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,31,0.009525333220760027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,31,0.009599999835093817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,31,0.009578666960199675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,63,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,63,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,63,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,63,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,127,0.011514666179815928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,127,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,127,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,127,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,255,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,255,0.01166933278242747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,255,0.011407999942700068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,255,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,511,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,511,0.011477333803971609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,511,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,511,0.012714666624863943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,1023,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,1023,0.017952000101407368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,1023,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,1023,0.017984000345071156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,2047,0.028079998989899952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,2047,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,2047,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,2047,0.02882666637500127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,4095,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,4095,0.047877331574757896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,4095,0.0498933345079422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,4095,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,8191,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,8191,0.08720533053080241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,8191,0.09081066648165385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,8191,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,16383,0.011306667079528173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,16383,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,16383,0.16733866930007935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,16383,0.17187732458114624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,1,0.008826666822036108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,1,0.008837333569924036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,32767,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,1,0.008101333553592363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,1,0.007941333577036858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,32767,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,32767,0.37142932415008545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,3,0.008693333094318708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,3,0.008645333349704742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,3,0.007882666463653246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,15,0.008736000085870424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,32767,0.3821280002593994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,3,0.008266666904091835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,7,0.00789866658548514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,7,0.008496000121037165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,15,0.008736000085870424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,15,0.008293333152929941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,15,0.007877333089709282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,31,0.009290666629870733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,31,0.009434666484594345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,31,0.008341333518425623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,31,0.008266666904091835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,63,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,63,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,63,0.008314666648705801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,63,0.008256000156203905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,127,0.011349332829316458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,127,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,127,0.00898133342464765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,127,0.009258666386206945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,255,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,255,0.012746666868527731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,255,0.00985599992175897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,255,0.010709332923094431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,511,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,511,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,511,0.00926399976015091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,511,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,1023,0.013269333789745966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,1023,0.00961599995692571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,1023,0.01328533391157786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,1023,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,2047,0.015178666760524115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,2047,0.015872000406185787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,2047,0.009589333087205887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,2047,0.012736000120639801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,4095,0.009306666751702627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,4095,0.01533866673707962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,4095,0.01624533285697301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,4095,0.014394666999578476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,8191,0.01924266666173935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,8191,0.024645333488782246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,8191,0.00933333362142245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,8191,0.01989866668979327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,16383,0.02037866661945979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,16383,0.02979733298222224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,16383,0.009829333052039146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,16383,0.02629866699377696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,1,0.008565333361426989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,32767,0.044031997521718345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,32767,0.009450666606426239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,32767,0.024618667860825855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,1,0.00850133349498113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,1,0.006495999793211619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,32767,0.03897066662708918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,3,0.006517333288987477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,1,0.006704000135262807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,3,0.006714666883150737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,3,0.008650666723648706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,7,0.008922666932145754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,3,0.008576000109314919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,7,0.00874133345981439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,7,0.006714666883150737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,7,0.006773333375652631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,15,0.008954666554927826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,15,0.008863999818762144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,31,0.009418666362762451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,15,0.006575999781489372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,15,0.006671999891599019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,31,0.009599999835093817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,31,0.0068800002336502075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,31,0.006853333363930385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,63,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,63,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,63,0.006927999978264173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,63,0.006821333120266597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,127,0.01126933346192042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,127,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,127,0.00744000015159448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,127,0.007760000104705493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,255,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,255,0.011306667079528173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,511,0.011685332904259363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,255,0.008250666782259941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,255,0.008912000184257826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,511,0.011861333002646765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,511,0.00797333319981893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,511,0.009119999905427298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,1023,0.011744000017642975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,1023,0.012069333344697952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,1023,0.008000000069538752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,2047,0.018709332992633183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,1023,0.00949866697192192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,2047,0.016330666840076447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,2047,0.008282666405042013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,2047,0.011781333635250727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,4095,0.016800000021855038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,4095,0.020527999848127365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,4095,0.008410666758815447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,4095,0.0138026662170887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,7,0.008682666967312494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,8191,0.02063999945918719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,7,0.00878399983048439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,8191,0.028378665447235107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,8191,0.008074666683872541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,8191,0.018709332992633183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,16383,0.023610666394233704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,16383,0.008352000266313553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,1,0.008703999842206636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,1,0.008517333616813024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,1,0.005888000130653381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,32767,0.02888533224662145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,1,0.005914666379491488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,32767,0.06195733447869619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,3,0.00867733359336853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,32767,0.008496000121037165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,32767,0.04380266865094503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,3,0.008496000121037165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,3,0.005968000118931134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,7,0.005834666391213735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,3,0.005925333127379417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,15,0.008767999708652496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,7,0.008869333192706108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,7,0.008943999807039896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,7,0.0058453331391016645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,15,0.008746666833758354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,15,0.005973333492875099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,15,0.00596266674498717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,31,0.009472000102202097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,31,0.009648000200589498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,31,0.005781333272655805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,31,0.006058666855096817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,63,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,63,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,63,0.00600533311565717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,63,0.006085333103934924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,127,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,127,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,127,0.006309333567818006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,127,0.006415999804933866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,255,0.011621333658695221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,255,0.011760000139474869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,255,0.007007999966541926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,255,0.007466666400432587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,511,0.01184533288081487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,511,0.011749333391586939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,511,0.007418666655818622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,511,0.00784533346692721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,1023,0.01621866722901662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,1023,0.01699200024207433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,1023,0.007338666667540868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,1023,0.009338666374484697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,2047,0.01884799947341283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,2047,0.007776000226537387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,2047,0.02386133372783661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,2047,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,4095,0.023621333142121632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,4095,0.03244800120592117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,4095,0.007386666412154834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,4095,0.014127999544143677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,8191,0.02754666656255722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,8191,0.04496533175309499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,8191,0.007717333113153775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,8191,0.022917332748572033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,16383,0.03364799916744232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,16383,0.06844266752401988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,16383,0.008112000301480293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,16383,0.03142933299144109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,1,0.07858666777610779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,32767,0.03465066601832708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,32767,0.09142399827639262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,32767,0.008112000301480293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,32767,0.05297600229581197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,1,0.07853333155314128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,1,0.07937600215276082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,1,0.07900799810886383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,3,0.07867733140786488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,3,0.0792746643225352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,3,0.07913066446781158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,3,0.07959466675917308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,7,0.08178133269151051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,7,0.08191999793052673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,7,0.08298666775226593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,7,0.08230400085449219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,15,0.08517866333325703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,15,0.08481066425641377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,15,0.08509866396586101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,16383,0.02550933261712392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,15,0.08587200442949931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,31,0.10825066765149434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,31,0.10750400026639302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,31,0.10814932982126872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,16383,0.039349332451820374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,31,0.10830932855606079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,63,0.1095413366953532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,63,0.10962133606274922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,63,0.10981866717338562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,63,0.10966933767000835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,127,0.11077333490053813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,127,0.11002666751543681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,127,0.11027200023333232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,127,0.11131200194358826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,255,0.11135466893513997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,255,0.13409066200256348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,255,0.11035199960072835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,255,0.1341386636098226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,511,0.11117866635322571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,511,0.11132267117500305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,511,0.2074026664098104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,511,0.21092800299326578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,1023,0.11104533076286316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,1023,0.11116266250610352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,1023,0.35385600725809735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,1023,0.3632213274637858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,2047,0.11106666922569275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,1,0.01759999990463257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,2047,0.1109920044740041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,1,0.017653333644072216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,2047,0.6611200173695883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,2047,0.6526026725769043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,1,0.008709333216150602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,7,0.017792000124851864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,3,0.01777600000301997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,7,0.017445333302021027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,3,0.01766933376590411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,3,0.0086666668454806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,3,0.00871999996403853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,15,0.00871999996403853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,7,0.008698666468262672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,7,0.00874133345981439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,15,0.017621333400408428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,31,0.008703999842206636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,15,0.017450666675964992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,15,0.00873066671192646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,31,0.01766933376590411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,31,0.017792000124851864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,31,0.008725333337982496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,63,0.017349333812793095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,127,0.017423999806245167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,63,0.01738133281469345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,63,0.008581333483258883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,63,0.008623999853928884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,127,0.01758933315674464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,127,0.00961599995692571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,127,0.009503999724984169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,255,0.032474666833877563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,255,0.032186667124430336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,255,0.016666666915019352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,255,0.019189332922299702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,511,0.04365866879622141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,511,0.048991998036702476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,511,0.017210666090250015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,511,0.027701333165168762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,1023,0.043882668018341064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,1023,0.06124266485373179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,1023,0.017157333592573803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,2047,0.04589866598447164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,1023,0.03542399903138479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,2047,0.08269333342711131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,2047,0.017263999829689663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,1,0.023887999355793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,2047,0.05610666672388712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,1,0.023733332753181458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,3,0.024271999796231587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,3,0.023989332218964893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,1,0.008016000191370646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,1,0.00789866658548514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,3,0.007989333321650824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,3,0.007893333211541176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,7,0.024304000039895374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,7,0.024170666933059692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,15,0.024138666689395905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,7,0.008181333541870117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,7,0.008026666939258575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,15,0.023717333873112995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,15,0.007967999825874964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,15,0.008042666440208754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,31,0.023930666347344715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,31,0.023717333873112995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,31,0.008010666817426682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,31,0.007760000104705493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,63,0.023904000719388325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,63,0.02387733260790507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,63,0.007829333345095316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,63,0.007749333356817563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,127,0.023610666394233704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,127,0.02348800003528595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,127,0.009573333586255709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,255,0.04365866879622141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,127,0.009701333319147428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,255,0.0439626673857371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,255,0.01590399940808614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,255,0.018565333137909572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,511,0.044533332188924156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,511,0.05373866856098175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,511,0.01595199977358182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,511,0.023599999646345775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,1023,0.04436799883842468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,1023,0.06768533090750377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,1023,0.015919999529918034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,1023,0.03533333291610082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,1,0.008826666822036108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,2047,0.04668800036112467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,2047,0.05638400216897329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,2047,0.09331199526786804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,2047,0.016149333367745083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,1,0.04085333396991094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,1,0.04032533367474874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,3,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,1,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,3,0.0405973345041275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,3,0.04028266668319702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,3,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,7,0.040394666294256844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,7,0.040778666734695435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,7,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,7,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,15,0.04045866678158442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,15,0.04077333211898804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,15,0.011328000575304031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,15,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,31,0.04068266600370407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,31,0.04035733391841253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,31,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,31,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,63,0.039818666875362396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,63,0.040021332601706185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,63,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,63,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,127,0.01302933320403099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,127,0.039877332746982574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,127,0.013242666920026144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,255,0.04188266893227895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,255,0.0422026664018631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,255,0.013237333546082178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,255,0.017583999782800674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,511,0.043280000487963356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,511,0.05570666491985321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,511,0.012938667088747025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,511,0.02619733413060506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,1023,0.043237333496411644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,1023,0.0781173308690389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,1023,0.013141332815090815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,1023,0.043493335445721946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,2047,0.045567999283472695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,2047,0.1265120009581248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,2047,0.012949333836634954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,2047,0.07580799857775371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,float16,1,0.15101866920789084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,1,0.15202132860819498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,fp8,1,0.1521813372770945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,1,0.15213867028554282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,float16,3,0.15238400300343832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,3,0.1511733333269755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,fp8,3,0.15338133772214255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,3,0.15240533153216043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,float16,7,0.15771200259526572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,7,0.15898133317629495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,fp8,7,0.1590720017751058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,7,0.16014400124549866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,float16,15,0.16394133369127908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,15,0.16389333208402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,fp8,15,0.16606400410334268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,15,0.16499200463294983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,float16,31,0.21064533789952597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,31,0.209114670753479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,fp8,31,0.210698664188385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,31,0.21224000056584677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,float16,63,0.21149333318074545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,63,0.2130133310953776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,fp8,63,0.21340266863505045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,63,0.2135146657625834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,float16,127,0.21590399742126465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,127,0.21432000398635864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,fp8,127,0.21627734104792276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,127,0.2145599921544393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,float16,255,0.2150933345158895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,1,0.011381333072980246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,255,0.2637386719385783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,fp8,255,0.21438399950663248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,255,0.26187199354171753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,float16,511,0.21637332439422607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,float16,1,0.02903999884923299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,511,0.40679999192555744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,fp8,511,0.21606399615605673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,1,0.029440000653266907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,fp8,1,0.015205333630243937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,511,0.41302935282389325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,1,0.01403733342885971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,float16,3,0.028965334097544353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,3,0.014192000031471252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,fp8,3,0.014378666877746582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,7,0.02902399996916453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,float16,7,0.029253333806991577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,7,0.014149333039919535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,fp8,7,0.01413333291808764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,127,0.039861333866914116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,float16,15,0.02943466603755951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,15,0.014159999787807465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,15,0.028965334097544353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,fp8,15,0.014208000153303146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,float16,31,0.029493334392706554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,3,0.029498666524887085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,31,0.029189333319664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,fp8,63,0.014256000518798828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,fp8,31,0.014384000251690546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,31,0.014245333770910898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,float16,127,0.029152000943819683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,float16,63,0.029152000943819683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,63,0.028959999481836956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,63,0.014256000518798828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,127,0.028864001234372456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,255,0.028005334238211315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,fp8,127,0.01626666635274887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,127,0.016341333587964375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,float16,255,0.054383998115857445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,255,0.05449600021044413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,fp8,255,0.02441066751877467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,float16,511,0.05532800157864889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,511,0.06601066887378693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,fp8,511,0.024432001014550526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,511,0.035642666121323906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,float16,1,0.04193066557248434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,float16,3,0.04241600135962168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,1,0.04146133363246918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,fp8,1,0.012703999876976013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,1,0.012661332885424295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,3,0.041402667760849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,fp8,3,0.012666666259368261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,3,0.012698666503032049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,float16,7,0.042223999897638954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,7,0.04251733422279358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,fp8,15,0.012725333372751871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,fp8,7,0.012479999413092932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,15,0.012624000509579977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,7,0.012746666868527731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,float16,15,0.041663999358812966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,15,0.04155199974775314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,31,0.012538666526476542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,float16,31,0.04208533465862274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,31,0.04243200023969015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,fp8,31,0.012730666746695837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,float16,63,0.04074666649103165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,fp8,63,0.012533333152532578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,63,0.012367999802033106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,127,0.04172799984614054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,float16,127,0.04126933217048645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,fp8,127,0.015599999576807022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,127,0.015376000354687372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,float16,255,0.044106667240460716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,255,0.04442666471004486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,float16,511,0.04451733330885569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,fp8,255,0.01526933287580808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,255,0.020506666352351505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,511,0.05725333094596863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,fp8,511,0.015253332753976187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,511,0.03232000023126602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,float16,1,0.07367999851703644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,1,0.07355733215808868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,fp8,1,0.01912533367673556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,1,0.018874666343132656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,float16,3,0.07356266677379608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,3,0.07438399891058604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,fp8,3,0.018976000448067982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,3,0.018992000569899876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,float16,7,0.0739573339621226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,7,0.07298133273919423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,fp8,7,0.018858666221300762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,7,0.019109333554903667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,float16,15,0.07363200187683105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,15,0.07358400026957194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,fp8,15,0.018992000569899876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,float16,31,0.07358933488527934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,15,0.01876266673207283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,31,0.07461866736412048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,fp8,31,0.019029332945744198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,31,0.018976000448067982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,float16,63,0.07300800085067749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,63,0.07256533205509186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,fp8,63,0.018709332992633183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,63,0.019823999454577763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,float16,127,0.0743146687746048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,127,0.07381866872310638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,fp8,127,0.022485333184401195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,127,0.02237333357334137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,float16,255,0.07868266602357228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,255,0.07874666651089986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,fp8,255,0.022757334013779957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,511,0.09875200192133586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,255,0.0322080006202062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,511,0.04911466439565023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,float16,511,0.07922666768232982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,fp8,511,0.02237333357334137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,float16,1,0.2980213363965352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,float16,1,0.2957599957784017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,fp8,1,0.2981013258298238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,fp8,1,0.298309326171875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,fp8,3,0.2983893354733785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,float16,3,0.2985066572825114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,float16,3,0.29843733708063763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,fp8,3,0.3004426757494609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,float16,7,0.3097440004348755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,63,0.041034666200478874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,float16,7,0.3097013235092163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,fp8,7,0.3125600020090739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,fp8,7,0.312394658724467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,float16,15,0.32472000519434613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,float16,15,0.32477333148320514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,fp8,15,0.3264480034510295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,fp8,15,0.32427199681599933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,float16,31,0.4155946572621663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,float16,63,0.420197327931722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,float16,31,0.41260266304016113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,float16,63,0.42031999429066974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,fp8,31,0.415669322013855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,fp8,31,0.41548800468444824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,fp8,63,0.4243679841359456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,fp8,63,0.4214400053024292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,float16,127,0.4261386791865031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,float16,127,0.42630934715270996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,fp8,127,0.423583984375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,fp8,127,0.4267306725184123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,float16,255,0.42510934670766193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,float16,1,0.052144000927607216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,float16,1,0.05304533243179321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,float16,255,0.5187306801478068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,fp8,255,0.42633068561553955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,fp8,1,0.019893333315849304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,fp8,255,0.5139466524124146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,fp8,1,0.020015999674797058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,fp8,3,0.02011200040578842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,float16,3,0.052000001072883606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,float16,7,0.05203733344872793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,float16,3,0.052298665046691895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,fp8,3,0.019909333437681198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,float16,15,0.053130666414896645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,float16,7,0.052095999320348106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,fp8,7,0.019946667055288952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,fp8,7,0.020031999796628952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,float16,31,0.05186666548252106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,float16,15,0.05190399785836538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,fp8,15,0.019968000551064808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,fp8,15,0.02012266715367635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,float16,31,0.05307200054327647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,fp8,31,0.01989866668979327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,float16,63,0.05115200082461039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,fp8,31,0.020143999407688778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,float16,63,0.05136533578236898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,fp8,63,0.01988799994190534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,fp8,63,0.020090666910012562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,float16,127,0.051407997806866966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,fp8,127,0.023317334552605946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,float16,127,0.05232533315817515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,fp8,127,0.023685333629449207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,float16,255,0.05464000006516775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,float16,255,0.05487999816735586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,fp8,255,0.023919999599456787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,fp8,255,0.03123733401298523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,float16,1,0.07555200159549713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,float16,1,0.07699200014273326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,fp8,1,0.02182399978240331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,fp8,1,0.021738665799299877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,float16,3,0.07595733304818471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,float16,3,0.07639466722806294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,fp8,3,0.021770666042963665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,fp8,3,0.0216799999276797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,float16,7,0.07707199951012929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,float16,7,0.07623466849327087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,fp8,7,0.02183466653029124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,float16,15,0.07642666498819987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,float16,15,0.07595733304818471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,fp8,15,0.021674667795499165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,fp8,15,0.022064000368118286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,float16,31,0.07631466786066692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,float16,31,0.07619200150171916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,fp8,31,0.0220320001244545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,fp8,31,0.02187199890613556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,float16,63,0.07457066575686137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,float16,63,0.07546666761239369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,fp8,63,0.021781332790851593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,fp8,63,0.021583999196688335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,float16,127,0.07716266810894012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,float16,127,0.07632533212502797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,fp8,127,0.026693334182103474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,fp8,127,0.027002667387326557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,float16,255,0.08281599978605907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,float16,255,0.08182933429876964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,fp8,255,0.026975999275843304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,fp8,255,0.040805332362651825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,float16,1,0.13649066289265951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,fp8,1,0.033600000043710075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,float16,1,0.1388266682624817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,fp8,1,0.033786666889985405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,float16,3,0.13716800014177957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,fp8,3,0.03364799916744232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,fp8,3,0.033546666304270424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,float16,7,0.13877333203951517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,fp8,7,0.03374933451414108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,float16,7,0.13741333285967508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,fp8,7,0.03356799980004629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,float16,15,0.13759467005729675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,float16,15,0.13870400190353394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,fp8,15,0.03367999941110611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,fp8,15,0.03369600077470144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,float16,31,0.13683199882507324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,float16,31,0.13854400316874185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,fp8,31,0.03359466542800268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,fp8,31,0.033615998923778534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,float16,63,0.13898133238156637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,float16,63,0.13824533422787985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,fp8,63,0.03303466737270355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,fp8,63,0.033333333830038704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,float16,127,0.14036267002423605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,float16,127,0.13974933822949728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,fp8,127,0.045253331462542214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,fp8,127,0.04462933540344238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,1,0.013898666948080063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,1,0.013983999689420065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,float16,255,0.14637333154678345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,fp8,255,0.06001066664854685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,1,0.013834666460752487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,fp8,255,0.04462933540344238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,float16,255,0.1466506620248159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,1,0.013679999858140945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,3,0.013898666948080063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,3,0.013717333475748697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,7,0.013999999811251959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,3,0.013679999858140945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,3,0.013610667238632837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,7,0.013999999811251959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,7,0.013973332941532135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,7,0.013936000565687815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,15,0.014015999933083853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,15,0.014064000298579534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,15,0.013850666582584381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,15,0.014032000054915747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,31,0.015301333119471868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,31,0.015125333021084467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,31,0.01509333277742068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,31,0.015125333021084467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,63,0.018394666413466137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,63,0.018506667266289394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,63,0.018618666877349217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,63,0.018394666413466137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,127,0.01877333347996076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,fp8,7,0.02176533391078313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,127,0.01865600049495697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,127,0.018624000251293182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,127,0.01855466639002164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,255,0.01850133389234543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,255,0.01886933296918869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,255,0.01869333287080129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,255,0.01855466639002164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,511,0.018789333601792652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,511,0.02207999924818675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,511,0.018789333601792652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,511,0.021888000269730885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,1023,0.01871466636657715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,2047,0.018687999496857326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,1023,0.03194666653871536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,float16,3,0.13759467005729675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,1023,0.0185759998857975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,1023,0.031845333675543465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,2047,0.05116266508897146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,4095,0.01868266612291336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,2047,0.01846933364868164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,2047,0.05179733534653982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,4095,0.01871466636657715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,8191,0.1691733400026957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,4095,0.0904266635576884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,4095,0.09145067135492961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,8191,0.018719999740521114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,8191,0.018592000007629395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,8191,0.16995733976364136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,16383,0.018672000616788864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,16383,0.01887999971707662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,16383,0.3251039981842041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,16383,0.3277386625607808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,32767,0.018709332992633183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,1,0.013962666193644205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,1,0.013797332843144735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,32767,0.018464000274737675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,1,0.00784533346692721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,1,0.007749333356817563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,3,0.013834666460752487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,32767,0.8586506843566895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,3,0.013882666826248169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,32767,0.7555359999338785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,15,0.014127999544143677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,3,0.008250666782259941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,3,0.007941333577036858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,15,0.008383999889095625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,7,0.01414399966597557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,7,0.013936000565687815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,7,0.008400000010927519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,15,0.013999999811251959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,15,0.008357333640257517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,63,0.018538666268189747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,31,0.015200000256299973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,31,0.015306666493415833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,31,0.008517333616813024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,31,0.00820266641676426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,63,0.018357332795858383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,63,0.008277333031098047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,63,0.008463999877373377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,255,0.018640000373125076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,127,0.020245333512624104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,127,0.018901333212852478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,127,0.009306666751702627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,127,0.00914666677514712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,255,0.01874133323629697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,255,0.009935999910036722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,255,0.010656000425418219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,1023,0.012741333494583765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,511,0.012863999853531519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,1023,0.009359999870260557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,511,0.013162666310866674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,511,0.009733333562811216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,511,0.010581333190202713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,1023,0.013621332744757334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,1023,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,2047,0.018122666825850803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,2047,0.02090666691462199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,2047,0.009957333405812582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,2047,0.014783999572197596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,4095,0.01870399961868922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,8191,0.022629333039124806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,8191,0.03373866776625315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,4095,0.023370665808518726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,4095,0.010026666646202406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,4095,0.017312000195185345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,16383,0.026704000929991405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,8191,0.009578666960199675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,8191,0.023925334215164185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,16383,0.04781866570313772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,16383,0.010288000106811523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,16383,0.04264533519744873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,1,0.013951999445756277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,32767,0.03298133363326391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,32767,0.07601599891980489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,1,0.01392000044385592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,1,0.006650666395823161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,32767,0.009973333527644476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,1,0.006730666384100914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,32767,0.060191998879114784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,3,0.01392000044385592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,3,0.013845333208640417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,3,0.0068853336075941724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,3,0.006645333642760913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,7,0.014282666146755219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,7,0.015216000378131866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,7,0.0069759997228781385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,15,0.006773333375652631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,7,0.006842666616042455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,15,0.013989333063364029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,15,0.0141546664138635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,15,0.0069440001000960665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,31,0.01522133375207583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,31,0.015119999647140503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,31,0.006810666372378667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,63,0.006874666859706243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,31,0.00684799998998642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,63,0.01846933364868164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,63,0.018768000106016796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,63,0.006634666894872983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,127,0.018810667097568512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,127,0.018746666610240936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,7,0.00789866658548514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,127,0.00754666638871034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,127,0.007573333258430163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,255,0.011733333269755045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,255,0.011792000383138657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,255,0.008394666636983553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,255,0.008890666688481966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,511,0.011535999675591787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,511,0.011962667107582092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,511,0.00810666692753633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,511,0.009114666531483332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,2047,0.019226666539907455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,1023,0.016410666207472484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,1023,0.01746133342385292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,2047,0.013173333058754602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,1023,0.00843733362853527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,4095,0.023717333873112995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,1023,0.010656000425418219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,2047,0.024266667664051056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,2047,0.008901333436369896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,4095,0.03276266654332479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,4095,0.008367999767263731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,4095,0.017583999782800674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,8191,0.027866666515668232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,8191,0.045594667394955955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,8191,0.00860799973209699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,8191,0.027786667148272198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,16383,0.03444266567627589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,16383,0.06906666855017345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,16383,0.008890666688481966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,16383,0.03957866628964742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,32767,0.034901333351929985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,1,0.010138666878143946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,1,0.01007466639081637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,32767,0.0930560032526652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,1,0.00596266674498717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,32767,0.009082666908701261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,3,0.010101333260536194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,32767,0.06598933537801106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,3,0.010053333515922228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,7,0.010431999961535135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,3,0.005941333249211311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,3,0.005978666866819064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,7,0.010010666524370512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,15,0.010133333504199982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,7,0.006186666587988536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,7,0.005904000252485275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,15,0.010421333213647207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,15,0.005999999741713206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,31,0.009989333028594652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,15,0.005978666866819064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,31,0.010512000570694605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,31,0.00625599982837836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,31,0.005930666501323382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,63,0.010298666854699453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,63,0.010106666634480158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,63,0.006362666686375936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,63,0.005946666623155276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,255,0.012186666329701742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,127,0.00996800015370051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,127,0.010522666076819101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,127,0.006282666698098183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,127,0.006319999694824219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,255,0.012202666451533636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,255,0.007269333427151044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,255,0.007733333234985669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,511,0.01643199970324834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,1023,0.02475733309984207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,511,0.01754133279124896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,1023,0.010458666831254959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,2047,0.027061333258946735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,511,0.007338666667540868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,511,0.00855466661353906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,1023,0.021418665846188862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,1023,0.007637333124876022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,2047,0.03596800069014231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,2047,0.008223999912540117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,2047,0.016837333639462788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,4095,0.03385599950949351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,4095,0.05357866485913595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,8191,0.03382933388153712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,4095,0.008186666915814081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,4095,0.021269333859284718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,8191,0.06730666756629944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,8191,0.008287999778985977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,8191,0.029824001093705494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,16383,0.03369066615899404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,16383,0.08922666311264038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,16383,0.008442666381597519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,16383,0.047194664676984154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,32767,0.03375466664632162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,32767,0.1334986686706543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,32767,0.008314666648705801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,32767,0.07947733501593272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,float16,1,0.5892053445180258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,float16,1,0.5844853321711222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,fp8,1,0.5890560150146484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,fp8,1,0.5889226595560709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,float16,3,0.5861920118331909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,float16,3,0.5863093137741089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,1,0.0058186668902635574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,fp8,3,0.5936693350474039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,fp8,3,0.5889013210932413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,float16,7,0.6134026845296224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,float16,7,0.6136106650034586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,fp8,7,0.6192426681518555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,fp8,7,0.6191039880116781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,float16,15,0.6394453446070353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,float16,15,0.6439199844996134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,fp8,15,0.6430453459421793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,fp8,15,0.6430826584498087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,float16,31,0.8204853534698486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,float16,31,0.8196372985839844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,fp8,31,0.825381358464559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,fp8,31,0.8254559834798177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,float16,63,0.8353493213653564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,float16,63,0.8321706453959147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,fp8,63,0.8376800219217936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,fp8,63,0.8377119700113932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,float16,127,0.8591626485188802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,float16,127,0.8589653174082438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,float16,1,0.09629866480827332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,fp8,127,0.8438986937204996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,float16,1,0.09595732887585957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,fp8,1,0.03628266602754593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,fp8,1,0.03633599976698557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,fp8,127,0.8480106989542643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,float16,3,0.09589333335558574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,float16,3,0.0956160028775533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,fp8,3,0.0365226666132609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,fp8,3,0.03623466690381368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,float16,7,0.09627733627955119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,float16,7,0.09533333778381348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,fp8,7,0.03663466622432073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,float16,15,0.09622933467229207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,fp8,7,0.03718400001525879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,float16,15,0.09589333335558574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,fp8,15,0.036303999523321785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,fp8,15,0.036288000643253326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,float16,31,0.09672533472379048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,float16,31,0.09617066383361816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,fp8,31,0.03689600030581156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,fp8,31,0.03614933292071024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,float16,63,0.09611200292905171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,float16,63,0.09513066212336223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,fp8,63,0.03643200049797694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,fp8,63,0.03636800001064936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,float16,127,0.09664000074068706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,float16,127,0.09659733374913533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,fp8,127,0.0468800018231074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,fp8,127,0.045781334241231285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,float16,1,0.14256532986958823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,float16,1,0.1418826679388682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,fp8,1,0.0439573327700297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,fp8,1,0.042821332812309265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,float16,3,0.14294399817784628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,float16,3,0.141866664091746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,fp8,3,0.04284800092379252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,fp8,3,0.04354133208592733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,float16,7,0.14297599593798319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,float16,7,0.14198399583498636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,fp8,7,0.044069334864616394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,fp8,7,0.042853335539499916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,float16,15,0.14351999759674072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,float16,15,0.1420960028966268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,fp8,15,0.04251733422279358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,fp8,15,0.0436160018046697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,float16,31,0.14437333742777506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,float16,31,0.14412800470987955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,fp8,31,0.04394666850566864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,fp8,31,0.04253333310286204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,float16,63,0.14409066239992777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,fp8,63,0.042447999119758606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,float16,63,0.14309333761533102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,fp8,63,0.043381333351135254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,float16,127,0.1460533340771993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,float16,127,0.14506666858990988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,fp8,127,0.05898666878541311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,fp8,127,0.05898666878541311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,fp8,1,0.0720960001150767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,float16,1,0.266213337580363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,float16,1,0.265338659286499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,fp8,1,0.07200533151626587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,float16,3,0.26469866434733075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,float16,3,0.26627200841903687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,fp8,3,0.07197333375612895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,fp8,3,0.07154133419195811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,float16,7,0.26661866903305054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,fp8,7,0.0718399981657664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,float16,7,0.26547733942667645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,fp8,7,0.0716480016708374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,float16,15,0.268559992313385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,float16,15,0.269322673479716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,fp8,15,0.07160533467928569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,fp8,15,0.07179733117421468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,float16,31,0.2709493239720662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,fp8,31,0.07145600020885468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,float16,31,0.2716053326924642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,fp8,31,0.07172800103823344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,float16,63,0.27244265874226886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,fp8,63,0.07100800176461537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,float16,63,0.271232008934021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,fp8,63,0.07127999762694041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,float16,127,0.27154133717219037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,float16,127,0.2720053394635518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,fp8,127,0.08467732866605122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,fp8,127,0.08413333694140117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,float16,1,1.187066634496053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,float16,1,1.1902399857838948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,fp8,1,1.1925919850667317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,fp8,1,1.192954699198405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,float16,3,1.194650650024414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,float16,3,1.192303975423177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,fp8,3,1.1999093691507976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,fp8,3,1.1976693471272786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,float16,7,1.2440426349639893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,float16,7,1.244586706161499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,fp8,7,1.246117353439331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,fp8,7,1.247440020243327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,float16,15,1.3099146684010823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,float16,15,1.31112535794576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,fp8,15,1.310762643814087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,fp8,15,1.3104373613993328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,float16,31,1.6686612764994304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,float16,31,1.6691892941792805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,fp8,31,1.6730027198791504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,fp8,31,1.6729493141174316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,float16,63,1.6925759315490723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,float16,63,1.6934666633605957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,float16,1,0.1880906621615092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,fp8,1,0.08941866954167683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,float16,1,0.1872053345044454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,fp8,63,1.7052906354268391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,fp8,1,0.08939733107884724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,float16,3,0.18784000476201376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,fp8,63,1.7029600143432617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,float16,3,0.18766399224599203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,fp8,3,0.0892639954884847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,fp8,3,0.08921600381533305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,float16,7,0.1880906621615092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,fp8,7,0.08931199709574382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,float16,7,0.18741333484649658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,fp8,7,0.0895146628220876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,float16,15,0.18871466318766275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,float16,15,0.18771199385325113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,fp8,15,0.08918399612108867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,fp8,15,0.08947199583053589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,float16,31,0.18793600797653198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,float16,31,0.1872533361117045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,fp8,31,0.08959999680519104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,fp8,31,0.08957866827646892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,float16,63,0.18554667631785074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,float16,63,0.18568533658981323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,fp8,63,0.08889599641164143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,fp8,63,0.08844266335169475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,float16,1,0.28189865748087567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,float16,1,0.28230400880177814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,fp8,1,0.09175999959309895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,fp8,1,0.09192533294359843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,float16,3,0.2843466599782308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,fp8,3,0.09140800436337788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,float16,3,0.2826826572418213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,fp8,3,0.09145599603652954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,fp8,7,0.09145067135492961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,float16,7,0.28307199478149414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,float16,7,0.2829386591911316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,fp8,7,0.09166399637858073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,float16,15,0.2846933404604594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,fp8,15,0.09112000465393066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,float16,15,0.283135990301768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,fp8,15,0.09139200051625569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,float16,31,0.28435200452804565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,fp8,31,0.0913866659005483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,float16,31,0.2844906648000081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,fp8,31,0.09169066945711772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,float16,63,0.2823466658592224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,fp8,63,0.09059733152389526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,float16,63,0.28142933050791424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,fp8,63,0.09104532996813457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,fp8,1,0.13768000404040018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,float16,1,0.5349973440170288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,fp8,1,0.13742400209108988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,float16,1,0.5329813162485758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,float16,3,0.5339413483937582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,float16,3,0.5355733235677084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,fp8,3,0.13659733533859253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,fp8,3,0.13724799950917563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,fp8,7,0.13649066289265951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,float16,7,0.5352693398793539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,float16,7,0.5385760068893433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,float16,15,0.5389493306477865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,fp8,15,0.13715733091036478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,float16,15,0.5372906525929769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,fp8,15,0.1365173359711965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,float16,31,0.539680004119873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,float16,31,0.5431520144144694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,fp8,31,0.13606933752695718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,1,0.013530666629473368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,1,0.013573333621025085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,1,0.0136266661187013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,1,0.013616000612576803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,3,0.013477332890033722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,float16,63,0.5342826843261719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,fp8,63,0.13571733236312866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,float16,63,0.534874677658081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,3,0.013487999637921652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,fp8,63,0.1355413297812144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,3,0.013701333353916803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,3,0.013770667215188345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,7,0.014005333185195923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,7,0.013845333208640417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,7,0.014106666048367819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,7,0.014090667168299357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,15,0.014533333480358124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,15,0.014607999473810196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,15,0.014538666854302088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,15,0.014522666732470194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,31,0.017808000246683758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,31,0.017877332866191864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,31,0.018031999468803406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,63,0.018079999834299088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,63,0.017994667092959087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,31,0.017845333864291508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,63,0.017935999979575474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,63,0.018133333573738735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,127,0.018207999567190807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,127,0.018138666947682697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,127,0.018112000077962875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,127,0.018138666947682697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,255,0.0182239996890227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,255,0.021530665457248688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,255,0.01842133328318596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,255,0.021562665700912476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,511,0.032032000521818794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,511,0.018191999445358913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,511,0.0312266672650973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,511,0.018309333672126133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,1023,0.018522666146357853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,1023,0.05120000243186951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,1023,0.018602666755517323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,1023,0.05190933247407278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,2047,0.018383999665578205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,2047,0.09310400485992432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,2047,0.0899679958820343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,2047,0.018415999909241993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,4095,0.018538666268189747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,4095,0.018394666413466137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,4095,0.173418660958608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,4095,0.16820265849431357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,8191,0.018325333793958027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,8191,0.01854933301607768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,8191,0.33126399914423627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,8191,0.33688000837961835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,fp8,7,0.1365653375784556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,16383,0.01855466639002164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,1,0.013797332843144735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,1,0.013663999736309052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,1,0.008186666915814081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,16383,0.018496000518401463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,1,0.007936000203092894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,3,0.013765333841244379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,3,0.013744000345468521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,3,0.008165333420038223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,16383,0.8685973485310873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,3,0.007962666451931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,16383,0.7773013114929199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,7,0.014085333794355392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,7,0.014240000396966934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,15,0.008154666672150293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,7,0.008522666369875273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,7,0.00810666692753633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,15,0.014544000228246054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,15,0.014639999717473984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,15,0.00789866658548514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,31,0.017877332866191864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,31,0.0182239996890227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,31,0.0084906667470932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,31,0.008005333443482717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,fp8,31,0.1367680033047994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,63,0.018079999834299088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,63,0.0180479995906353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,63,0.008303999900817871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,63,0.008576000109314919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,127,0.018309333672126133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,127,0.01830400029818217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,127,0.008922666932145754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,127,0.009343999748428663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,255,0.013237333546082178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,255,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,255,0.009573333586255709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,511,0.010575999816258749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,255,0.010239999741315842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,511,0.013162666310866674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,511,0.01339200014869372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,511,0.009429333110650381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,2047,0.02186133215824763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,1023,0.018533332894245785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,1023,0.019589333484570186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,2047,0.010687999427318573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,1023,0.009919999788204828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,1023,0.012960000584522883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,2047,0.027679999669392902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,2047,0.017781333376963932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,4095,0.026837334036827087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,4095,0.03771200031042099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,4095,0.010037333394090334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,4095,0.026314665873845417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,8191,0.03201599915822347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,8191,0.03648533423741659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,8191,0.05494933327039083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,8191,0.010405333091815313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,1,0.010389333590865135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,16383,0.03937600056330363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,16383,0.0825973351796468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,1,0.010245333115259806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,16383,0.010703999549150467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,1,0.006751999879876773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,1,0.006784000123540561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,16383,0.06358399987220764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,7,0.010314666976531347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,3,0.010117333382368088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,3,0.010165333126982054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,3,0.006906666482488315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,3,0.006671999891599019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,7,0.010373333469033241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,7,0.006704000135262807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,7,0.006773333375652631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,15,0.010181333248813948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,15,0.01009599988659223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,15,0.006917333230376244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,15,0.006853333363930385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,31,0.01033599985142549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,31,0.010399999717871347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,31,0.006821333120266597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,31,0.006725333631038666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,63,0.009994666402538618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,63,0.010255999863147736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,63,0.006831999868154526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,63,0.007045333584149678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,127,0.010149333626031876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,127,0.009893333539366722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,127,0.007231999809543292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,127,0.007071999832987785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,255,0.012245333443085352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,255,0.012149333953857422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,255,0.008207999790708223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,255,0.008725333337982496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,511,0.016650666793187458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,511,0.017237332959969837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,511,0.008565333361426989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,2047,0.027552001178264618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,511,0.009866666669646898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,1023,0.02179199953873952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,1023,0.024773334463437397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,1023,0.00890666681031386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,1023,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,2047,0.03621333340803782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,2047,0.009418666362762451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,4095,0.02649066597223282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,2047,0.02025066688656807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,4095,0.03443733354409536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,4095,0.05342400074005127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,4095,0.009194666519761086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,8191,0.03438399980465571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,8191,0.06797333558400472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,8191,0.00938666673998038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,8191,0.036714665591716766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,16383,0.03425066669782003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,16383,0.08969600001970927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,1,0.010399999717871347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,16383,0.00927466650803884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,1,0.010693332801262537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,16383,0.0568800022204717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,1,0.005957333371043205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,1,0.006111999973654747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,3,0.010245333115259806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,3,0.010656000425418219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,3,0.006037333359320958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,3,0.006213333457708359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,7,0.010464000205198923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,7,0.010597333312034607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,7,0.006186666587988536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,15,0.00596266674498717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,7,0.005914666379491488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,15,0.010629333555698395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,15,0.010453333457310995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,15,0.006288000072042148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,31,0.010693332801262537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,31,0.010618666807810465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,63,0.006111999973654747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,31,0.00596266674498717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,31,0.005893333504597346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,63,0.010543999572594961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,63,0.010373333469033241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,63,0.005984000240763028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,255,0.018266666680574417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,127,0.010426666587591171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,127,0.010527999450763067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,127,0.0064319999267657595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,127,0.006522666662931442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,255,0.018122666825850803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,255,0.007647999872763951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,255,0.008373333141207695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,511,0.022656001150608063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,511,0.025253333151340485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,511,0.008330666770537695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,511,0.0099093330403169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,1023,0.03305600086847941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,1023,0.04118400067090988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,1023,0.008736000085870424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,2047,0.020527999848127365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,1023,0.016688000410795212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,2047,0.03530666728814443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,2047,0.050767997900644936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,2047,0.008746666833758354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,4095,0.03622400015592575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,4095,0.06578133503595988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,4095,0.008672000219424566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,4095,0.02871999889612198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,8191,0.03535466641187668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,8191,0.08806932965914409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,8191,0.00877333308259646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,8191,0.04665066798528036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,16383,0.034847999612490334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,16383,0.13345066706339517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,16383,0.008805333326260248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,16383,0.07856533428033192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,1,0.022826666633288067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,1,0.02363733450571696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,3,0.024090667565663654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,15,0.030975999931494396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,3,0.02443733314673106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,31,0.031354665756225586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,63,0.0313226655125618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,7,0.024890666206677754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,31,0.031258667508761086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,7,0.02514133354028066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,15,0.03142933299144109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,63,0.03143999973932902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,127,0.0376800000667572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,127,0.03755199909210205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,255,0.057130664587020874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,255,0.05739733576774597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,511,0.09564266602198283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,1023,0.17382399241129556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,1023,0.17457600434621176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,2047,0.3263733386993408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,2047,0.3295360008875529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,4095,0.647648016611735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,511,0.09669867157936096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,4095,0.6444106499354044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,8191,1.287295977274577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,8191,1.3018346627553303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,1,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,3,0.012309333930412928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,7,0.012448000411192576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,31,0.012554666648308435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,63,0.012261333564917246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,1,0.009423999736706415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,15,0.01250133290886879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,3,0.009632000078757605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,127,0.012213333199421564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,127,0.010149333626031876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,7,0.009296000003814697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,255,0.015930666277805965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,255,0.012223999947309494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,511,0.023749334116776783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,511,0.014570667097965876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,15,0.009461333354314169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,4095,0.06578133503595988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,31,0.009423999736706415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,1023,0.03444266567627589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,63,0.009466666728258133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,2047,0.05601066847642263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,8191,0.08331733445326488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,4095,0.08532800277074178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,1023,0.026538667579491932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,8191,0.11049600442250569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,2047,0.03860266755024592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,3,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,31,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,1,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,63,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,7,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,1,0.00730666642387708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,127,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,127,0.008042666440208754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,255,0.02178666740655899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,255,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,15,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,3,0.007376000285148621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,511,0.02922666569550832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,511,0.01825599993268649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,1023,0.04819199939568838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,7,0.007776000226537387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,1023,0.027658666173617046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,2047,0.06654933094978333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,2047,0.03602133442958196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,4095,0.08807466427485149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,4095,0.05212800204753876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,15,0.007360000163316727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,31,0.007311999797821045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,63,0.007471999774376552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,8191,0.1288106640179952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,8191,0.07718933125336964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,1,0.007109333450595538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,7,0.007178666690985362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,31,0.007002666592597961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,15,0.007018666714429855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,3,0.006858666737874349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,63,0.007040000210205714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,127,0.00810666692753633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,3,0.016250666230916977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,1,0.015850666910409927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,511,0.04749333361784617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,63,0.015935999651749928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,1023,0.06228800117969513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,127,0.015802666544914246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,255,0.030661332110563915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,1023,0.02903466671705246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,2047,0.04284800092379252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,2047,0.08803199728329976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,4095,0.12763733665148416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,4095,0.06677333513895671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,1,0.00878399983048439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,8191,0.11311466495196025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,8191,0.21857066949208578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,1,0.008799999952316284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,3,0.00890666681031386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,7,0.008799999952316284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,3,0.008863999818762144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,7,0.008943999807039896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,15,0.009002666920423508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,15,0.009018666421373686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,31,0.009621333330869675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,31,0.009722666814923286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,63,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,63,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,127,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,127,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,255,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,255,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,511,0.01579733317097028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,511,0.015722667177518208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,1023,0.02731200059254964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,1023,0.02720533311367035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,2047,0.03453333427508672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,2047,0.034117333590984344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,4095,0.04780800143877665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,31,0.015957333147525787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,4095,0.04698666433493296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,8191,0.07520533104737599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,8191,0.07336000104745229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,7,0.015658666690190632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,16383,0.12589333454767862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,16383,0.1286240021387736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,1,0.008725333337982496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,1,0.009029333169261614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,32767,0.2354080080986023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,32767,0.23040000597635904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,3,0.00884799969693025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,3,0.008890666688481966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,15,0.016063999384641647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,7,0.008703999842206636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,255,0.015658666690190632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,31,0.009098666409651438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,7,0.009141333401203156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,15,0.008752000207702318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,15,0.009701333319147428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,31,0.009658666948477427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,63,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,511,0.015775999675194424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,63,0.00955200009047985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,127,0.011407999942700068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,127,0.010543999572594961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,255,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,511,0.021802666286627453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,255,0.012304000556468964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,511,0.012703999876976013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,1023,0.02749866743882497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,1023,0.014229333649079004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,2047,0.03461866577466329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,8191,0.02204799900452296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,2047,0.015418666104475657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,16383,0.024458666642506916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,4095,0.01852799952030182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,4095,0.018181333939234417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,8191,0.02254933367172877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,16383,0.026021334032217663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,1,0.008816000074148178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,32767,0.03030399978160858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,1,0.007071999832987785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,32767,0.032933334509531655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,3,0.008912000184257826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,3,0.007050666958093643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,7,0.008890666688481966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,7,0.007344000041484833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,15,0.007071999832987785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,63,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,63,0.007327999919652939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,127,0.011365332951148352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,127,0.0075519997626543045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,255,0.011429333438475927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,255,0.009573333586255709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,511,0.01603200038274129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,511,0.00980266680320104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,1023,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,1023,0.02755733331044515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,2047,0.014922666052977243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,2047,0.01180800050497055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,4095,0.015599999576807022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,4095,0.013317332913478216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,8191,0.019018666197856266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,8191,0.016186666985352833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,16383,0.021509334444999695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,16383,0.01912533367673556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,1,0.008997333546479544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,3,0.008858666444818178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,3,0.0063733334342638654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,32767,0.03557866563399633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,32767,0.02719466636578242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,7,0.00890666681031386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,15,0.009045333291093508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,31,0.009690666571259499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,31,0.006144000217318535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,15,0.006213333457708359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,63,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,63,0.006234666953484218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,127,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,127,0.006831999868154526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,255,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,255,0.007925333455204964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,511,0.01584533353646596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,511,0.008256000156203905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,1023,0.01341333364446958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,1023,0.008709333216150602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,2047,0.014794666320085526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,2047,0.009685333197315535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,4095,0.016437333077192307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,4095,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,8191,0.02510933329661687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,8191,0.014815999815861383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,15,0.009072000160813332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,16383,0.01829333355029424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,16383,0.030661332110563915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,31,0.009503999724984169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,1,0.009119999905427298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,32767,0.04987733562787374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,1,0.00898133342464765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,3,0.00884799969693025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,3,0.009098666409651438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,7,0.008933333059151968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,32767,0.025519999365011852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,7,0.009002666920423508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,15,0.009119999905427298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,31,0.007386666412154834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,15,0.009375999992092451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,31,0.00985599992175897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,127,0.011594666788975397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,31,0.009935999910036722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,63,0.011365332951148352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,63,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,127,0.011584000041087469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,255,0.011637333780527115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,255,0.011653333902359009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,511,0.02603200078010559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,511,0.02554133286078771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,1023,0.02979733298222224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,1023,0.030005333324273426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,2047,0.04033066580692927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,2047,0.04121066629886627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,4095,0.06106133262316386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,4095,0.05991466840108236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,8191,0.10072533289591472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,1,0.006693333387374878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,8191,0.09940800070762634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,16383,0.18081067005793253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,16383,0.17921066284179688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,1,0.008933333059151968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,1,0.009301333377758661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,32767,0.34093332290649414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,32767,0.3349013328552246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,3,0.00902399979531765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,3,0.009178666397929192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,7,0.006389333556095759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,7,0.008949333180983862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,7,0.009296000003814697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,63,0.009541333342591921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,15,0.009109333157539368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,15,0.00897066667675972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,31,0.009882666791478792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,31,0.00919999989370505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,63,0.011535999675591787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,511,0.012762666990359625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,127,0.011567999919255575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,127,0.010581333190202713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,255,0.01157333329319954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,2047,0.018144000321626663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,255,0.012149333953857422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,511,0.025914666553338368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,1023,0.030048000315825146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,1023,0.013829333086808523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,2047,0.015642666568358738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,4095,0.01905599981546402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,4095,0.018079999834299088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,8191,0.023189333577950794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,8191,0.021722666919231415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,16383,0.026309333741664886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,3,0.008933333059151968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,32767,0.03822399924198786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,16383,0.026314665873845417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,32767,0.04496533175309499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,1,0.008976000050703684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,1,0.0069866664707660675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,3,0.007674666742483775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,7,0.009056000038981438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,7,0.007114666824539502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,15,0.00919999989370505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,127,0.007690666864315669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,15,0.007290666922926903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,255,0.011727999895811081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,31,0.009743999689817429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,511,0.025962665677070618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,31,0.006927999978264173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,63,0.011541333049535751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,63,0.0074346667776505155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,127,0.011482667177915573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,255,0.009429333110650381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,511,0.00980266680320104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,1023,0.013653332988421122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,1023,0.010458666831254959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,2047,0.01569066693385442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,2047,0.011786667009194693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,4095,0.016773333152135212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,4095,0.013466666142145792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,8191,0.018325333793958027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,1,0.009098666409651438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,16383,0.03190399954716364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,3,0.0063146669417619705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,16383,0.021744000415007275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,32767,0.052229334910710655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,32767,0.03540800015131632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,1,0.006165333092212677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,3,0.009194666519761086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,31,0.006186666587988536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,7,0.008874666566650072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,7,0.006293333445986112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,15,0.009189333145817121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,127,0.00690133310854435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,15,0.0061919999619325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,31,0.009946666657924652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,63,0.011701333026091257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,63,0.006224000205596288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,127,0.011461333682139715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,8191,0.02629333237806956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,2047,0.02025066688656807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,255,0.011701333026091257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,255,0.008016000191370646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,511,0.013162666310866674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,511,0.008341333518425623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,1023,0.013663999736309052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,1023,0.008933333059151968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,2047,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,4095,0.023728000621000927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,16383,0.05352533360322317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,4095,0.01301866645614306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,8191,0.035349334279696144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,1,0.04186666508515676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,8191,0.017077332983414333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,16383,0.02977599948644638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,1,0.04275199770927429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,32767,0.08148266871770223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,7,0.04566933214664459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,32767,0.04257066547870636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,3,0.04372266431649526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,3,0.04411733150482178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,7,0.04504533112049103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,15,0.05753066639105479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,15,0.05675200124581655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,31,0.05721066892147064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,31,0.05691733459631602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,63,0.05815466741720835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,63,0.05752533177534739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,127,0.06982933481534322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,127,0.07037333150704701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,255,0.10801066954930623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,255,0.10803733269373576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,511,0.1841920018196106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,511,0.18648000558217367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,1023,0.33581864833831787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,1023,0.3370826641718547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,2047,0.6482453346252441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,2047,0.6407413482666016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,1,0.013269333789745966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,1,0.00956266683836778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,3,0.013253333667914072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,3,0.009749333063761393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,7,0.013151999562978745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,4095,1.2540480295817058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,7,0.00938666673998038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,15,0.01313599944114685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,4095,1.2608906428019206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,15,0.009488000224033991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,31,0.01351999988158544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,31,0.009530666594703993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,63,0.013045333325862885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,255,0.014101333916187286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,63,0.009461333354314169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,127,0.013013333082199097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,127,0.010368000095089277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,255,0.025989333788553875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,511,0.034917332231998444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,511,0.024559999505678814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,1023,0.0591839998960495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,1023,0.04530133306980133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,2047,0.08460799853006999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,2047,0.0655626654624939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,4095,0.10863999525705974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,4095,0.08316799998283386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,7,0.008080000057816505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,1,0.016042667130629223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,1,0.0081386665503184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,3,0.01609066625436147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,3,0.008000000069538752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,7,0.016421332955360413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,15,0.01634666696190834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,63,0.007941333577036858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,15,0.008090666805704435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,31,0.015967999895413715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,31,0.008101333553592363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,63,0.015957333147525787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,511,0.048656001687049866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,127,0.01632000009218852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,127,0.009178666397929192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,255,0.03148799886306127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,255,0.018378666291634243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,511,0.026730666557947796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,1023,0.06397333244482677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,1023,0.03756266583998998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,2047,0.08771199981371562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,1,0.02589333305756251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,2047,0.052058666944503784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,4095,0.1301866670449575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,4095,0.07728533446788788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,1,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,3,0.025477332373460133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,3,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,7,0.025989333788553875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,7,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,15,0.0259253333012263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,15,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,127,0.013082666943470636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,31,0.025626666843891144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,31,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,63,0.025301332275072735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,63,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,127,0.026186667382717133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,255,0.05175999800364176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,255,0.021642667551835377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,511,0.06572799881299336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,511,0.029450667401154835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,1023,0.08522133032480876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,1023,0.042805333932240806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,2047,0.13205867012341818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,1,0.008421333506703377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,2047,0.06765866776307423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,4095,0.2239840030670166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,7,0.008623999853928884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,1,0.008538666491707167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,4095,0.11434666315714519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,3,0.008538666491707167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,3,0.008687999720374743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,7,0.008650666723648706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,15,0.009253333633144697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,15,0.00955200009047985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,31,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,31,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,63,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,63,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,127,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,127,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,255,0.013114667187134424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,1023,0.028058665494124096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,2047,0.04783466458320618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,255,0.012896000097195307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,511,0.017925333231687546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,511,0.018031999468803406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,1023,0.027786667148272198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,2047,0.04842133323351542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,4095,0.0867680013179779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,4095,0.0881760021050771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,8191,0.16519999504089355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,8191,0.16856000820795694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,16383,0.32365334033966064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,16383,0.324618657430013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,1,0.008597333605090777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,1,0.009279999881982803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,3,0.008592000231146812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,3,0.00919999989370505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,32767,0.9050453503926595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,7,0.0085333331177632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,32767,0.7670933405558268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,7,0.009637333452701569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,15,0.009322666873534521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,15,0.009594666461149851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,31,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,31,0.009136000027259191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,63,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,63,0.009568000212311745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,127,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,127,0.010512000570694605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,255,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,255,0.01258133351802826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,511,0.01782400036851565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,511,0.012768000364303589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,1023,0.015775999675194424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,1023,0.013781332721312841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,2047,0.01926933353145917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,2047,0.015583999454975128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,4095,0.020661332954963047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,4095,0.018058666338523228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,8191,0.033946665624777474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,8191,0.02513599892457326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,16383,0.04057066639264425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,16383,0.030954666435718536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,1,0.008506666868925095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,1,0.007242666557431221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,3,0.00871999996403853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,32767,0.06572266419728597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,32767,0.05830933153629303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,3,0.007135999699433644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,7,0.00850133349498113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,7,0.00701333334048589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,15,0.009466666728258133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,15,0.009413333609700203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,127,0.007823999971151352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,31,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,31,0.007525333513816197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,63,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,63,0.007418666655818622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,1023,0.010741333166758219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,127,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,255,0.012928000340859095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,255,0.009621333330869675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,511,0.013242666920026144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,511,0.00985599992175897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,1023,0.013738666971524557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,2047,0.02186133215824763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,2047,0.013248000293970108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,4095,0.025397333006064098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,4095,0.015658666690190632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,8191,0.03788800040880839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,8191,0.020949333906173706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,16383,0.05600533386071523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,16383,0.037418665985266365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,32767,0.08520000179608662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,32767,0.05301866432030996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,1,0.0084906667470932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,1,0.00620266670982043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,3,0.00873066671192646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,3,0.006213333457708359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,7,0.008703999842206636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,7,0.006464000170429547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,15,0.009402666861812273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,15,0.006298666819930077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,31,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,31,0.006640000268816948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,63,0.01139733319481214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,63,0.006239999706546466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,511,0.008496000121037165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,127,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,1023,0.01027199998497963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,127,0.006805333619316419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,255,0.012826666235923767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,255,0.00810666692753633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,4095,0.040847999354203544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,511,0.013269333789745966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,1023,0.019674666225910187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,2047,0.028079998989899952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,8191,0.05924266576766968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,2047,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,4095,0.020207999895016353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,16383,0.04692266881465912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,8191,0.026719999810059864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,16383,0.09025599559148152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,1,0.07825600107510884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,1,0.0800906668106715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,32767,0.06898666421572368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,32767,0.1309599975744883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,3,0.08196266492207845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,15,0.10839999715487163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,15,0.10874133308728536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,3,0.08284266789754231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,7,0.08471999565760295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,7,0.08619733651479085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,31,0.1088266670703888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,31,0.10898133118947347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,63,0.1097920040289561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,63,0.11028800408045451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,127,0.13354133566220602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,127,0.13296533624331155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,255,0.20800000429153442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,255,0.20814400911331177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,511,0.3595199982325236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,511,0.35741865634918213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,1023,0.6645706494649252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,1,0.02027200038234393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,1023,0.6536639928817749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,3,0.0206133338312308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,1,0.010602666685978571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,3,0.010378666842977205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,7,0.020138667275508244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,15,0.010496000448862711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,7,0.010458666831254959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,15,0.020421333611011505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,2047,1.2581760088602703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,31,0.020341333001852036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,2047,1.2556479771931965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,31,0.010330666477481524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,255,0.039642666776975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,63,0.020394666741291683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,63,0.010288000106811523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,127,0.02029866725206375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,127,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,255,0.02372266600529353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,511,0.06098666787147522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,511,0.044346665342648826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,1023,0.07972800234953563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,1023,0.06585599978764851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,2047,0.11181333661079407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,1,0.02677333354949951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,2047,0.08249066770076752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,1,0.012784000486135483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,3,0.026816000541051228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,3,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,7,0.02703999976317088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,7,0.013023999830087027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,15,0.026730666557947796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,15,0.01267733300725619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,31,0.026447998980681103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,31,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,63,0.02664533257484436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,63,0.013002666334311167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,127,0.02699200063943863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,127,0.015333333363135656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,255,0.05314133564631144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,255,0.026565333207448322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,511,0.06749866902828217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,1,0.045194665590922035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,511,0.039290666580200195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,1023,0.08755200107892354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,1023,0.052853330969810486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,2047,0.1360319952170054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,2047,0.07780266801516215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,1,0.015274666249752045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,3,0.04527466495831808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,3,0.015205333630243937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,31,0.045456002155939736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,7,0.04554666578769684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,7,0.01540800059835116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,15,0.046367997924486794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,15,0.015498666713635126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,31,0.01545599972208341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,63,0.044938668608665466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,63,0.015205333630243937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,127,0.04615999758243561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,127,0.01903466631968816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,255,0.05575466652711233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,255,0.026928000152111053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,511,0.07600533465544383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,511,0.04055466751257578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,1023,0.12133866548538208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,1023,0.06460799773534139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,2047,0.21655466159184775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,2047,0.11300266782442729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,1,0.15159466862678528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,1,0.15448533495267233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,3,0.15729600191116333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,3,0.16019200285275778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,15,0.2118933399518331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,7,0.16448533535003662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,7,0.1673120061556498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,15,0.21292799711227417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,31,0.21205333868662515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,31,0.21071465810139975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,63,0.21386667092641196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,63,0.215013325214386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,127,0.26049067576726276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,127,0.2602986693382263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,255,0.41092268625895184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,255,0.40854398409525555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,1,0.03469333300987879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,1,0.017301333447297413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,511,0.7098133563995361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,3,0.034517332911491394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,3,0.01738133281469345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,511,0.7051146825154623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,7,0.03480000048875809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,7,0.017530667285124462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,15,0.03419200082619985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,15,0.01766933376590411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,31,0.03446399917205175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,31,0.01730666682124138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,63,0.034341332813103996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,63,0.01754133279124896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,127,0.03482666611671448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,127,0.019648000597953796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,255,0.06760533154010773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,255,0.05264533559481303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,511,0.08505599697430928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,511,0.06705066561698914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,1,0.04797866443792979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,1,0.018778666853904724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,3,0.04647466540336609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,3,0.018538666268189747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,7,0.047194664676984154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,7,0.01865600049495697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,15,0.04665066798528036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,15,0.018650667121013004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,31,0.04779199759165446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,31,0.018394666413466137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,63,0.04632000128428141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,63,0.018533332894245785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,127,0.04974933465321859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,127,0.023872000475724537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,255,0.058693334460258484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,1,0.02740799884001414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,255,0.03516799956560135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,511,0.07898133496443431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,1,0.08342933654785156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,511,0.0487360010544459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,3,0.08391466736793518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,3,0.02624533325433731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,7,0.0830026666323344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,31,0.0836906631787618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,7,0.026341333985328674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,15,0.0836906631787618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,15,0.025968000292778015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,31,0.026159999271233875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,63,0.084906667470932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,63,0.02630399912595749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,127,0.08565333485603333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,127,0.03736000011364619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,255,0.0491839994986852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,255,0.09954133629798889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,511,0.0706826647122701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,511,0.1355946660041809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,1,0.29367466767628986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,fp8,1,0.30352532863616943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,3,0.3090826670328776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,fp8,3,0.31484800577163696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,7,0.3240586717923482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,fp8,7,0.3319946726163228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,15,0.41759467124938965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,fp8,15,0.41570667425791424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,31,0.4177493254343669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,fp8,31,0.41908268133799237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,fp8,63,0.4217333396275838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,63,0.4230453173319499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,127,0.524069348971049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,fp8,127,0.51910932858785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,1,0.06237866481145223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,fp8,1,0.032842665910720825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,255,0.8188426494598389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,fp8,255,0.817039966583252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,7,0.06322666505972545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,3,0.06333333253860474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,fp8,3,0.03313600023587545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,fp8,7,0.03366400053103765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,31,0.06371733546257019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,15,0.06283733248710632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,fp8,15,0.032698666055997215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,fp8,31,0.03264000018437704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,fp8,63,0.0332640012105306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,63,0.06444266438484192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,127,0.06613333523273468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,fp8,127,0.04287466903527578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,255,0.0772213339805603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,fp8,255,0.062234664956728615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,1,0.08656000097592671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,fp8,1,0.035829332967599235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,3,0.08694400389989217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,fp8,3,0.03586133321126302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,15,0.08773866295814514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,7,0.08760533730189006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,fp8,7,0.035375999907652535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,fp8,15,0.036117332677046456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,31,0.0883893370628357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,63,0.08785067001978557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,fp8,31,0.03548266738653183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,127,0.08998933434486389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,fp8,63,0.03682666768630346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,fp8,127,0.05539200206597646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,255,0.10619200269381206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,fp8,1,0.05789866546789805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,fp8,255,0.06541866560777028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,1,0.15593600273132324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,7,0.1564640005429586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,3,0.15707199772198996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,fp8,3,0.05890666445096334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,fp8,7,0.05821333328882853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,15,0.1586026648680369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,fp8,15,0.058592001597086586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,fp8,31,0.05801066756248474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,31,0.15933866302172342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,63,0.15874666968981424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,fp8,63,0.05834666887919108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,127,0.1596213380495707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,1,0.01370666672786077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,fp8,127,0.06824533144632976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,1,0.013754667093356451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,3,0.013712000101804733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,fp8,255,0.0869653324286143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,255,0.18646933635075888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,3,0.01402666668097178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,7,0.013999999811251959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,31,0.01851733277241389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,7,0.01381333296497663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,15,0.014970666418472925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,15,0.015135999768972397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,31,0.0183146670460701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,63,0.018511999398469925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,63,0.018709332992633183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,127,0.0186666672428449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,127,0.018581333259741466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,255,0.021984001000722248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,255,0.02204799900452296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,511,0.031983998914559685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,511,0.031930667658646904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,1023,0.0513973335425059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,1023,0.05170666674772898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,4095,0.16835200786590576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,2047,0.08975999553998311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,2047,0.09065066774686177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,4095,0.1683839956919352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,8191,0.3224853277206421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,8191,0.3238346576690674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,1,0.013530666629473368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,1,0.009098666409651438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,3,0.014032000054915747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,3,0.009136000027259191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,7,0.01404800017674764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,7,0.009119999905427298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,16383,0.6324426730473837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,16383,0.6459199984868368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,15,0.01516266663869222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,15,0.009119999905427298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,31,0.01820266619324684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,31,0.009136000027259191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,63,0.01850133389234543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,255,0.012383999923865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,63,0.009749333063761393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,511,0.015087999403476715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,127,0.018725333114465077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,127,0.0100426667680343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,255,0.021925332645575207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,32767,1.7180372873942058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,32767,1.7899573644002278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,511,0.012714666624863943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,1023,0.015610666324694952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,1023,0.01402666668097178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,2047,0.025205334027608235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,2047,0.01739199956258138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,4095,0.030613332986831665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,4095,0.020741333564122517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,8191,0.04608533283074697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,8191,0.03613866617282232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,16383,0.06856533388296764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,16383,0.05099200208981832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,1,0.013471999516089758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,1,0.007098666702707608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,3,0.013850666582584381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,3,0.007045333584149678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,32767,0.10546132922172546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,32767,0.08386666576067607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,7,0.01404800017674764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,7,0.007216000308593114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,63,0.0074560002734263735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,15,0.014938666174809137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,15,0.0075093333919843035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,31,0.0182239996890227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,31,0.007770666852593422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,63,0.01852799952030182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,127,0.01858666663368543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,127,0.00790933333337307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,255,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,255,0.009503999724984169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,511,0.013269333789745966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,511,0.010138666878143946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,1023,0.019402666638294857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,1023,0.012053333222866058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,2047,0.0284853329261144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,2047,0.015077333897352219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,8191,0.032773333291212715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,4095,0.041237334410349526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,4095,0.02458133300145467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,8191,0.060175999999046326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,16383,0.09067199627558391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,16383,0.05629333357016245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,3,0.010559999694426855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,1,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,1,0.006319999694824219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,3,0.006271999950210254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,32767,0.13404267032941183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,7,0.010533332824707031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,32767,0.08144000172615051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,7,0.0063573333124319715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,15,0.010650667051474253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,15,0.0063040001938740415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,31,0.010538666198650995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,31,0.006704000135262807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,63,0.010608000059922537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,63,0.006106666599710782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,127,0.010501333822806677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,127,0.0069440001000960665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,255,0.014597332725922266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,255,0.008266666904091835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,2047,0.043162668744723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,511,0.020101333657900494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,511,0.009824000298976898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,1023,0.02826133370399475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,1023,0.015530666957298914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,2047,0.019173332800467808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,4095,0.06806399921576183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,4095,0.029317334294319153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,8191,0.08662933111190796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,8191,0.04387733340263367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,16383,0.12983999649683634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,16383,0.06807999809583028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,32767,0.2181653380393982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,32767,0.11469333370526631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,1,0.596725344657898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,fp8,1,0.6157600084940592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,3,0.6215146780014038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,fp8,3,0.6316213210423788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,7,0.6588426828384399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,fp8,7,0.6654666662216187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,15,0.8397546609242758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,31,0.8558613459269205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,fp8,31,0.8389120101928711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,63,0.8597280184427897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,fp8,63,0.8544053236643473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,127,1.0469919840494792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,1,0.12422933181126912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,fp8,1,0.09171733260154724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,fp8,127,1.0439893404642742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,3,0.12230400244394939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,fp8,3,0.09159466624259949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,7,0.12317867080370586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,fp8,7,0.09109866619110107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,15,0.12457600235939026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,fp8,15,0.09175466497739156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,fp8,31,0.09116799632708232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,31,0.12312533458073933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,63,0.12342400352160136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,fp8,63,0.09132267038027446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,127,0.1232266624768575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,fp8,127,0.10222400228182475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,fp8,1,0.0830080012480418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,1,0.16785067319869995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,3,0.16708266735076904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,fp8,3,0.0832479993502299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,7,0.16800000270207724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,fp8,7,0.08346133430798848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,15,0.16933866341908774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,fp8,15,0.08294400076071422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,fp8,31,0.08341333270072937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,31,0.16905067364374796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,63,0.16752533117930093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,fp8,63,0.08308266599973042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,127,0.168778657913208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,fp8,127,0.10041067004203796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,fp8,1,0.10788800319035848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,1,0.30822400252024335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,fp8,3,0.10807466506958008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,3,0.31037867069244385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,7,0.31194667021433514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,fp8,7,0.1090666651725769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,fp8,15,0.10751466949780782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,15,0.3103040059407552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,fp8,31,0.10819733142852783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,31,0.31320534149805707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,63,0.3109813332557678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,fp8,63,0.10748266180356343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,fp8,127,0.12778666615486145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,127,0.3098026712735494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,float16,1,1.2328960100809734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,fp8,1,1.259717305501302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,float16,3,1.2535093625386555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,fp8,15,0.8334346612294515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,fp8,3,1.2876640160878499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,float16,7,1.3282079696655273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,fp8,7,1.3515733083089192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,float16,15,1.6828319231669109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,float16,1,0.2341066598892212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,fp8,15,1.6762720743815105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,float16,31,1.6949440638224285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,fp8,31,1.680938720703125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,float16,63,1.7132107416788738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,fp8,63,1.7046507199605305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,fp8,1,0.1777706742286682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,float16,3,0.23374400536219278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,fp8,3,0.17763733863830566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,float16,7,0.23452800512313843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,fp8,7,0.17855999867121378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,float16,15,0.2339359919230143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,fp8,15,0.1774453322092692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,float16,31,0.23356799284617105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,fp8,31,0.1781066656112671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,float16,63,0.2336533268292745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,fp8,63,0.17753599087397257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,fp8,1,0.1569546659787496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,float16,1,0.3261599938074748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,fp8,3,0.15682133038838705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,float16,3,0.32596800724665326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,float16,7,0.32635732491811115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,fp8,7,0.15708266695340475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,float16,15,0.32554133733113605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,fp8,15,0.15685866276423135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,fp8,31,0.15607466300328574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,float16,31,0.32849599917729694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,fp8,63,0.15517866611480713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,float16,63,0.3240373333295186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,fp8,1,0.20820266008377075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,float16,1,0.612613320350647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,fp8,3,0.20811732610066733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,float16,3,0.6126773357391357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,fp8,7,0.20811200141906738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,float16,7,0.6128213405609131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,1,0.013557333499193192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,fp8,15,0.20828799406687418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,3,0.014010666559139887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,float16,15,0.6136800050735474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,1,0.013594667116800943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,3,0.013999999811251959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,7,0.01461333284775416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,float16,31,0.6170293490091959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,7,0.014378666877746582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,fp8,31,0.2076693375905355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,15,0.01766933376590411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,fp8,63,0.2059839963912964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,15,0.017653333644072216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,31,0.018906666586796444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,float16,63,0.6110453208287557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,31,0.017871999492247898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,63,0.017866666118303936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,63,0.01786133274435997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,511,0.052149335543314614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,127,0.021253332495689392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,127,0.021269333859284718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,1023,0.09267200032869975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,255,0.031093334158261616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,255,0.031445334355036415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,511,0.05109333495299021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,1023,0.09038933118184407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,2047,0.16873600085576376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,2047,0.17418134212493896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,4095,0.3264960050582886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,4095,0.33350932598114014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,1,0.013536000003417334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,8191,0.6635253429412842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,1,0.009365333244204521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,8191,0.6598186492919922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,3,0.013877333452304205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,3,0.009354666496316591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,7,0.014384000251690546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,7,0.009189333145817121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,15,0.017877332866191864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,15,0.009957333405812582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,31,0.017935999979575474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,127,0.021354667842388153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,127,0.010618666807810465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,31,0.009402666861812273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,16383,1.722885290781657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,63,0.01803733284274737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,63,0.00926399976015091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,16383,1.862170696258545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,255,0.014922666052977243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,255,0.012074666718641916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,511,0.01534933348496755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,511,0.012789333860079447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,1023,0.023573334018389385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,4095,0.052069331208864846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,1023,0.01573866605758667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,2047,0.03386666625738144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,8191,0.05821866790453593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,2047,0.026335999369621277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,4095,0.03165333221356074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,8191,0.07458666463692983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,16383,0.1127893328666687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,1,0.007402666533986728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,1,0.010415999839703241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,16383,0.08802666266759236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,3,0.010522666076819101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,3,0.007285333548982938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,7,0.010565333068370819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,7,0.0074560002734263735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,15,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,15,0.008031999692320824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,31,0.0106133334338665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,31,0.007397333160042763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,63,0.010687999427318573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,63,0.007167999943097432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,127,0.010581333190202713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,127,0.007983999947706858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,1023,0.028304000695546467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,255,0.013610667238632837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,255,0.009712000067035357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,511,0.01951466624935468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,511,0.011306667079528173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,1023,0.019253333409627277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,2047,0.0444213350613912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,2047,0.023183998962243397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,4095,0.06758933266003926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,4095,0.03926933308442434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,8191,0.08718933661778767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,8191,0.05357333521048228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,1,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,16383,0.13011733690897623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,16383,0.0765066643555959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,1,0.006341333190600078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,15,0.006389333556095759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,3,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,3,0.0063146669417619705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,7,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,63,0.0064106664309899015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,7,0.0063573333124319715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,127,0.007242666557431221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,15,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,31,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,31,0.006335999816656113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,63,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,127,0.012122667084137598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,255,0.02107200026512146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,255,0.00927466650803884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,511,0.028309332827727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,511,0.015285332997639975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,1023,0.046762665112813316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,1023,0.021685334543387096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,2047,0.06543999910354614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,2047,0.028362666567166645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,4095,0.08501332998275757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,4095,0.043007999658584595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,8191,0.12661866346995035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,8191,0.06704000135262807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,16383,0.21662400166193643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,1,0.013904000322024027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,16383,0.11313600341478984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,3,0.008197333042820295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,1,0.013647999614477158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,7,0.014266667266686758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,1,0.008031999692320824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,7,0.008272000278035799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,7,0.008378666515151659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,1,0.007978666573762894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,15,0.014538666854302088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,3,0.013839999834696451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,3,0.013936000565687815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,3,0.008186666915814081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,7,0.014096000542243322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,15,0.014639999717473984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,15,0.008394666636983553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,15,0.00795199970404307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,31,0.017829333742459614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,31,0.01801066721479098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,31,0.0080960001796484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,31,0.008240000034372011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,63,0.018021332720915478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,63,0.01800000046690305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,63,0.008586666857202848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,255,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,255,0.013866666704416275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,63,0.007871999715765318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,255,0.00961599995692571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,127,0.018160000443458557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,127,0.01828266680240631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,127,0.009029333169261614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,127,0.009445333232482275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,511,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,255,0.010591999938090643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,511,0.012800000607967377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,511,0.01351999988158544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,511,0.009306666751702627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,1023,0.018511999398469925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,2047,0.028309332827727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,1023,0.01970133309563001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,1023,0.010245333115259806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,2047,0.017551999539136887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,1023,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,2047,0.021946666141351063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,2047,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,4095,0.026922665536403656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,4095,0.027653334041436512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,4095,0.038218667109807335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,4095,0.010191999996701876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,8191,0.031898667414983116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,8191,0.0537120004494985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,8191,0.010421333213647207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,8191,0.0366293340921402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,16383,0.04025600105524063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,16383,0.0831573357184728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,1,0.010175999874869982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,1,0.006842666616042455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,16383,0.010559999694426855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,3,0.010250666489203772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,1,0.010117333382368088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,16383,0.06250666578610738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,1,0.006768000001708667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,3,0.01020800011853377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,7,0.006560000280539195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,3,0.006810666372378667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,3,0.006965333595871925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,7,0.010106666634480158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,7,0.010015999898314476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,31,0.010117333382368088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,7,0.006741333131988843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,15,0.010154666379094124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,15,0.0099093330403169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,15,0.007109333450595538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,15,0.0068853336075941724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,31,0.010138666878143946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,31,0.0069226666043202085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,127,0.00996800015370051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,31,0.006746666505932808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,63,0.010138666878143946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,63,0.009813333551088968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,255,0.012186666329701742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,63,0.007807999849319458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,63,0.007018666714429855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,127,0.009989333028594652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,127,0.007407999907930692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,127,0.007333333293596904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,255,0.011930666863918304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,255,0.00808533343176047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,255,0.008762666955590248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,511,0.01714133347074191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,511,0.01657066618402799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,511,0.00871999996403853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,511,0.009786666681369146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,1023,0.02205866575241089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,1023,0.024458666642506916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,1023,0.009072000160813332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,1023,0.013056000073750814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,2047,0.0276053324341774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,2047,0.036229332288106285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,2047,0.009445333232482275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,2047,0.019893333315849304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,4095,0.03465066601832708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,4095,0.05421866476535797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,4095,0.009226666763424873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,4095,0.025994665920734406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,8191,0.03457066665093104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,8191,0.06809066732724507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,16383,0.09036266803741455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,8191,0.009354666496316591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,8191,0.03654933224121729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,16383,0.03472533325354258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,1,0.010277333358923594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,16383,0.00922133338948091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,1,0.01027199998497963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,16383,0.058389330903689064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,1,0.006085333103934924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,1,0.006224000205596288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,3,0.010293333480755487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,3,0.010405333091815313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,3,0.0060159998635451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,3,0.005888000130653381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,15,0.01032533310353756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,7,0.010106666634480158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,7,0.01044800008336703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,7,0.006042666733264923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,7,0.006170666466156642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,15,0.010170666500926018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,15,0.006341333190600078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,15,0.006101333225766818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,63,0.010079999764760336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,31,0.010239999741315842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,31,0.010117333382368088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,127,0.010255999863147736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,31,0.006101333225766818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,31,0.006197333335876465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,63,0.010346666599313417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,63,0.0060159998635451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,63,0.006053333481152852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,127,0.010170666500926018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,127,0.006458666796485583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,127,0.006549333532651265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,255,0.018239999810854595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,255,0.018005333840847015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,255,0.007658666620651881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,255,0.008261333530147871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,511,0.02253866692384084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,511,0.02483733246723811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,511,0.008549333239595095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,511,0.009952000031868616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,1023,0.016693333784739178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,1023,0.033189333975315094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,1023,0.03934400031963984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,1023,0.00879466657837232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,2047,0.03532266616821289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,2047,0.052058666944503784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,2047,0.008874666566650072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,2047,0.020554666717847187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,4095,0.035418666899204254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,4095,0.06612800061702728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,4095,0.00860799973209699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,8191,0.008837333569924036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,4095,0.029333333174387615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,8191,0.03551999976237615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,16383,0.008762666955590248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,8191,0.08845866719881694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,8191,0.04609066744645437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,16383,0.03519999980926514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,16383,0.13260266184806824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,16383,0.07858666777610779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,1,0.006784000123540561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,1,0.006298666819930077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,3,0.0064319999267657595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,7,0.0064266665528217954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,7,0.006405333057045937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,3,0.0063680000603199005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,1,0.01463466634353002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,1,0.014256000518798828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,3,0.014170666535695394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,31,0.014111999422311783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,3,0.014277332772811254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,31,0.014159999787807465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,7,0.01431999976436297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,7,0.014720000326633453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,63,0.014325333138306936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,63,0.014479999740918478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,63,0.006319999694824219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,63,0.006458666796485583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,15,0.014538666854302088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,127,0.014245333770910898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,127,0.014565333724021912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,127,0.007002666592597961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,127,0.006831999868154526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,255,0.02658133457104365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,15,0.014458666245142618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,255,0.02651199946800868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,255,0.008240000034372011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,255,0.00960533320903778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,511,0.04029866556326548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,511,0.03589333345492681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,511,0.009754666437705358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,511,0.015717333803574245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,1023,0.036090667049090065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,1023,0.04961599906285604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,1023,0.009877333417534828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,2047,0.038480001191298165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,1023,0.019567999988794327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,2047,0.06690133114655812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,2047,0.02779199928045273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,2047,0.009973333527644476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,4095,0.03808533400297165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,4095,0.09010666608810425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,4095,0.009904000287254652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,4095,0.0447573314110438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,8191,0.03818133225043615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,15,0.0064853330453236895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,8191,0.1337493360042572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,8191,0.00980266680320104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,8191,0.07675733168919881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,1,0.007946666950980822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,16383,0.038202665746212006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,15,0.006218666831652324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,16383,0.009743999689817429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,1,0.007962666451931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,1,0.007887999837597212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,16383,0.14139733711878458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,16383,0.2243679960568746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,1,0.007776000226537387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,3,0.0075519997626543045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,3,0.007989333321650824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,3,0.00786666696270307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,7,0.007584000006318092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,3,0.007957333077987036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,7,0.007861333588759104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,7,0.008303999900817871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,7,0.00785600021481514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,15,0.007631999750932057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,15,0.007903999959429106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,31,0.006175999840100606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,15,0.008058666562040647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,15,0.008303999900817871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,31,0.006330666442712148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,31,0.0080960001796484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,31,0.007887999837597212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,31,0.007887999837597212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,31,0.007674666742483775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,63,0.008645333349704742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,63,0.009008000294367472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,63,0.007711999739209811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,63,0.007765333478649457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,255,0.00898133342464765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,127,0.00867733359336853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,127,0.008997333546479544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,127,0.008757333581646284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,127,0.009093333035707474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,255,0.008752000207702318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,255,0.009253333633144697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,255,0.010682666053374609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,511,0.008799999952316284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,511,0.010058666889866194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,511,0.009253333633144697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,511,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,2047,0.013306666165590286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,1023,0.013679999858140945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,1023,0.015754666179418564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,1023,0.009237333511312803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,1023,0.011584000041087469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,2047,0.016837333639462788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,2047,0.009178666397929192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,2047,0.013248000293970108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,8191,0.014458666245142618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,4095,0.01972266659140587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,4095,0.027471999327341717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,4095,0.009653333574533463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,4095,0.014730667074521383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,8191,0.01691199963291486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,8191,0.009285333255926767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,8191,0.017450666675964992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,16383,0.015429332852363586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,16383,0.0205226664741834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,16383,0.018207999567190807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,16383,0.009301333377758661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,32767,0.0173333336909612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,32767,0.022181332111358643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,32767,0.009226666763424873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,32767,0.02624000112215678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,1,0.008047999814152718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,1,0.007887999837597212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,1,0.006634666894872983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,1,0.006597333277265231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,3,0.007914666707317034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,3,0.00785600021481514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,3,0.006640000268816948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,3,0.006602666651209195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,7,0.007807999849319458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,65535,0.018960000326236088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,65535,0.02552533398071925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,7,0.008021333565314611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,7,0.006597333277265231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,7,0.006528000036875407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,15,0.007733333234985669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,65535,0.009413333609700203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,65535,0.033071999748547874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,15,0.00786666696270307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,31,0.006575999781489372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,15,0.006634666894872983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,15,0.006655999769767125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,31,0.008272000278035799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,31,0.008010666817426682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,31,0.006650666395823161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,63,0.00879466657837232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,63,0.008752000207702318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,63,0.006586666529377301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,63,0.006602666651209195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,127,0.008943999807039896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,127,0.008912000184257826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,127,0.007301333049933116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,511,0.009925333162148794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,127,0.007290666922926903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,255,0.008976000050703684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,255,0.009183999771873156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,255,0.007877333089709282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,255,0.008506666868925095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,511,0.008986666798591614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,511,0.00797333319981893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,511,0.009039999917149544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,1023,0.013658666362365087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,1023,0.015824000040690105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,1023,0.008287999778985977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,1023,0.009509333098928133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,2047,0.013221333424250284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,2047,0.01691199963291486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,2047,0.007930666829148928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,4095,0.012448000411192576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,2047,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,4095,0.012597333639860153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,8191,0.015210667004187902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,4095,0.013290667285521826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,4095,0.007893333211541176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,8191,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,8191,0.007791999727487564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,8191,0.014959999670584997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,16383,0.017557332913080852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,16383,0.007989333321650824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,16383,0.014197333405415217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,16383,0.016490666816631954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,32767,0.01515199989080429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,32767,0.01911466692884763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,32767,0.007781333600481351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,32767,0.022592000663280487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,1,0.007802666475375493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,1,0.008053333188096682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,1,0.00578666664659977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,1,0.006111999973654747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,3,0.007696000238259633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,65535,0.017594666530688603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,3,0.007738666608929634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,3,0.005984000240763028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,3,0.006448000048597653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,65535,0.022474666436513264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,7,0.007807999849319458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,65535,0.008074666683872541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,7,0.007733333234985669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,7,0.00625599982837836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,65535,0.028202667832374573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,7,0.005765333150823911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,31,0.007989333321650824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,15,0.007834666719039282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,15,0.007967999825874964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,15,0.006277333324154218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,15,0.005775999898711841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,31,0.008021333565314611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,31,0.005850666513045629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,31,0.005781333272655805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,63,0.009093333035707474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,63,0.008922666932145754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,63,0.006122666721542676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,63,0.005754666402935982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,127,0.008853333070874214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,127,0.00884799969693025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,255,0.007002666592597961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,127,0.0063040001938740415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,127,0.006277333324154218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,255,0.009002666920423508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,255,0.009066666786869368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,255,0.0075519997626543045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,511,0.008938666433095932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,511,0.00979200005531311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,511,0.007146666447321574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,511,0.008016000191370646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,1023,0.013829333086808523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,1023,0.015770666301250458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,2047,0.007296000296870868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,1023,0.007258666679263115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,1023,0.008320000022649765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,2047,0.012047999848922094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,4095,0.007055999711155891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,2047,0.012650666137536367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,2047,0.00916800027092298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,4095,0.012629333883523941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,4095,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,4095,0.010293333480755487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,8191,0.013199999928474426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,16383,0.016965333372354507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,8191,0.015487999965747198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,8191,0.007205333560705185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,16383,0.014271999398867289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,8191,0.01219733307758967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,16383,0.013877333452304205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,16383,0.007327999919652939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,32767,0.017103999853134155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,32767,0.0262719988822937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,32767,0.007162666569153468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,32767,0.019973333925008774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,1,0.007760000104705493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,1,0.00795199970404307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,1,0.005888000130653381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,65535,0.018965333700180054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,3,0.008026666939258575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,1,0.005781333272655805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,65535,0.032218667368094124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,3,0.007813333223263422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,65535,0.0074879998962084455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,3,0.0058186668902635574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,65535,0.024800000091393787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,3,0.005765333150823911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,7,0.007946666950980822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,7,0.007834666719039282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,7,0.0058666666348775225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,7,0.00589866687854131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,15,0.007914666707317034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,15,0.00789866658548514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,15,0.00589866687854131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,15,0.0063573333124319715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,31,0.007983999947706858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,31,0.008181333541870117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,31,0.005882666756709416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,31,0.006026666611433029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,127,0.008954666554927826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,127,0.008858666444818178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,63,0.008703999842206636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,255,0.008896000062425932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,63,0.008938666433095932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,63,0.006346666564544042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,63,0.005749333028992017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,127,0.006250666454434395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,127,0.006319999694824219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,255,0.008938666433095932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,255,0.007050666958093643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,255,0.00766933336853981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,511,0.00915733352303505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,511,0.009994666402538618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,511,0.007130666946371396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,511,0.007685333490371704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,1023,0.01173866664369901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,1023,0.011695999652147293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,1023,0.00720000018676122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,1023,0.008181333541870117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,2047,0.012576000144084295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,2047,0.012794667234023413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,2047,0.007087999954819679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,2047,0.00916800027092298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,4095,0.01209066684047381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,4095,0.01351999988158544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,4095,0.007151999821265538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,4095,0.010298666854699453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,8191,0.015674666812022526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,16383,0.016634666671355564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,8191,0.02021866664290428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,8191,0.007125333572427432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,8191,0.013125333935022354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,32767,0.02006400004029274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,16383,0.02383466561635335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,16383,0.007189333438873291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,16383,0.016117333124081295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,32767,0.03501333296298981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,32767,0.007125333572427432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,32767,0.022453332940737408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,1,0.007920000081261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,1,0.007871999715765318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,65535,0.023904000719388325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,1,0.007786666974425316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,1,0.007770666852593422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,3,0.007877333089709282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,3,0.007989333321650824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,3,0.007840000092983246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,3,0.007653333246707916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,65535,0.00749333327015241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,65535,0.05397333204746246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,7,0.007882666463653246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,65535,0.03301866600910822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,15,0.008176000167926153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,7,0.007978666573762894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,7,0.007903999959429106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,7,0.00772266648709774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,15,0.00786666696270307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,15,0.008341333518425623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,15,0.008389333263039589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,31,0.008090666805704435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,31,0.008005333443482717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,31,0.00785600021481514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,31,0.007765333478649457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,63,0.008789333204428354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,63,0.00898133342464765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,63,0.007802666475375493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,63,0.008325333396593729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,127,0.009056000038981438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,127,0.008912000184257826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,127,0.008885333314538002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,127,0.009072000160813332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,255,0.009301333377758661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,255,0.008943999807039896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,255,0.00921066664159298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,255,0.010602666685978571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,511,0.009189333145817121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,1023,0.009722666814923286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,1023,0.011973333855470022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,511,0.009861333295702934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,2047,0.02740799884001414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,511,0.00926399976015091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,511,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,1023,0.014032000054915747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,1023,0.01579733317097028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,2047,0.02056533346573512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,2047,0.00938666673998038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,2047,0.013125333935022354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,4095,0.014607999473810196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,4095,0.015168000012636185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,4095,0.009248000259200731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,4095,0.015066667149464289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,8191,0.015487999965747198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,8191,0.018181333939234417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,8191,0.009568000212311745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,8191,0.017466666797796886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,16383,0.016917333006858826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,16383,0.020517333100239437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,16383,0.018992000569899876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,16383,0.009445333232482275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,32767,0.018016000588734944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,32767,0.0235359991590182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,32767,0.009077333534757296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,32767,0.026447998980681103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,1,0.008000000069538752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,1,0.007840000092983246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,1,0.006511999915043513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,1,0.006538666784763336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,3,0.007903999959429106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,3,0.007818666597207388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,3,0.006730666384100914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,65535,0.02014933278163274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,65535,0.009216000015536943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,3,0.006586666529377301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,7,0.007840000092983246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,7,0.006629333520929019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,65535,0.027642667293548584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,65535,0.033359999457995095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,7,0.007877333089709282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,7,0.006474666918317477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,15,0.007903999959429106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,15,0.007781333600481351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,15,0.006597333277265231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,15,0.006730666384100914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,31,0.008367999767263731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,31,0.008127999802430471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,31,0.006640000268816948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,31,0.00679466687142849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,63,0.008703999842206636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,63,0.008810666700204214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,63,0.006720000257094701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,63,0.0069226666043202085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,127,0.009093333035707474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,127,0.008997333546479544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,127,0.0074346667776505155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,127,0.007893333211541176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,255,0.008938666433095932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,255,0.008949333180983862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,255,0.008127999802430471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,255,0.008826666822036108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,511,0.00891733355820179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,511,0.0099093330403169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,511,0.008037333066264788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,511,0.00926399976015091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,1023,0.013904000322024027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,1023,0.015834666788578033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,1023,0.00797333319981893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,1023,0.00984533317387104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,2047,0.012736000120639801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,2047,0.013424000392357508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,2047,0.00814933329820633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,2047,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,4095,0.013354666531085968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,4095,0.013882666826248169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,4095,0.007920000081261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,4095,0.012165332833925882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,8191,0.014010666559139887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,8191,0.01599466676513354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,8191,0.007882666463653246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,8191,0.014794666320085526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,16383,0.007930666829148928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,16383,0.014815999815861383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,16383,0.017418666432301205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,16383,0.017498667041460674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,32767,0.018005333840847015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,32767,0.027002667387326557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,32767,0.007903999959429106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,32767,0.024688000480333965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,1,0.007978666573762894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,1,0.007861333588759104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,1,0.005770666524767876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,3,0.008010666817426682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,1,0.00596266674498717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,3,0.00785600021481514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,3,0.005712000032265981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,65535,0.020010666300853092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,65535,0.03057066599527995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,3,0.006101333225766818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,65535,0.03311999887228012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,65535,0.007936000203092894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,7,0.007962666451931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,7,0.007871999715765318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,7,0.005946666623155276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,7,0.00589866687854131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,15,0.007770666852593422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,15,0.008005333443482717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,15,0.005946666623155276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,15,0.006031999985376994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,31,0.008261333530147871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,31,0.008112000301480293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,31,0.00590933362642924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,31,0.005829333638151486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,63,0.008576000109314919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,63,0.008837333569924036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,127,0.006351999938488007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,63,0.006037333359320958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,63,0.00600533311565717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,127,0.008933333059151968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,127,0.008863999818762144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,127,0.0064319999267657595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,255,0.009045333291093508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,255,0.008805333326260248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,255,0.007333333293596904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,255,0.007962666451931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,511,0.008954666554927826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,511,0.009952000031868616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,511,0.007242666557431221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,511,0.008000000069538752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,1023,0.011642667154471079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,2047,0.013056000073750814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,1023,0.012293333808581034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,1023,0.007216000308593114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,1023,0.008522666369875273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,4095,0.01403733342885971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,2047,0.012842666357755661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,2047,0.007333333293596904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,2047,0.009178666397929192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,4095,0.012773333738247553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,4095,0.007098666702707608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,4095,0.010362666721145311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,8191,0.016271999726692837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,8191,0.020538666596015293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,8191,0.007194666812817256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,8191,0.013477332890033722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,16383,0.01709866647919019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,16383,0.02459733436505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,16383,0.017557332913080852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,16383,0.007226666435599327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,32767,0.02022933339079221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,32767,0.03585066646337509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,32767,0.007130666946371396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,32767,0.023013333479563396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,1,0.008016000191370646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,1,0.007733333234985669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,1,0.005946666623155276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,1,0.005904000252485275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,3,0.007743999982873599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,3,0.007962666451931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,65535,0.02454400062561035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,3,0.005914666379491488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,65535,0.054901331663131714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,3,0.005888000130653381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,7,0.008010666817426682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,65535,0.0074560002734263735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,65535,0.03512533257404963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,7,0.007882666463653246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,7,0.005914666379491488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,15,0.005834666391213735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,7,0.006079999729990959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,15,0.00790933333337307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,15,0.00797333319981893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,15,0.006037333359320958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,31,0.008330666770537695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,31,0.008143999924262365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,31,0.005850666513045629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,31,0.007296000296870868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,63,0.008762666955590248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,63,0.00890666681031386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,127,0.0063680000603199005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,63,0.006074666976928711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,63,0.00624533308049043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,255,0.009093333035707474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,127,0.008885333314538002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,127,0.008874666566650072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,127,0.0063146669417619705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,255,0.009114666531483332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,255,0.00720000018676122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,255,0.007541333635648091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,511,0.011690666278203329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,511,0.011530666301647821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,511,0.006954666847983996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,511,0.007631999750932057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,1023,0.011754666765530905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,1023,0.011952000359694162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,1023,0.007221333061655362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,1023,0.008320000022649765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,2047,0.014949332922697067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,2047,0.007114666824539502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,2047,0.01752000053723653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,2047,0.009674666449427605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,4095,0.016010666886965435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,8191,0.007120000198483467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,4095,0.019541333119074505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,4095,0.007327999919652939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,4095,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,16383,0.03731200098991394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,8191,0.018794666975736618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,8191,0.026965332527955372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,8191,0.014869333555301031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,16383,0.022005334496498108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,16383,0.007333333293596904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,32767,0.058965335289637245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,16383,0.01974933346112569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,32767,0.02794666588306427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,32767,0.007418666655818622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,32767,0.03469866762558619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,1,0.011674666156371435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,1,0.011754666765530905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,1,0.0081386665503184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,1,0.008021333565314611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,3,0.01157333329319954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,3,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,3,0.008506666868925095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,65535,0.03417066733042399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,3,0.008512000242869059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,7,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,7,0.011557333171367645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,65535,0.09177066882451375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,15,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,65535,0.00766933336853981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,65535,0.05163733164469401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,7,0.0085333331177632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,7,0.008527999743819237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,15,0.011482667177915573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,15,0.008373333141207695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,31,0.008522666369875273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,15,0.008469333251317343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,31,0.011338666081428528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,31,0.011488000551859537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,63,0.008485333373149237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,31,0.00855466661353906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,63,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,63,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,63,0.008517333616813024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,127,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,127,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,255,0.009626666704813639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,127,0.009269333134094873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,127,0.008976000050703684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,255,0.013477332890033722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,255,0.013957332819700241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,255,0.010245333115259806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,511,0.019215999792019527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,511,0.019861333072185516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,511,0.010239999741315842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,511,0.011936000237862269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,1023,0.024959998826185863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,1023,0.02889599899450938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,1023,0.010480000327030817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,1023,0.020448000480731327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,2047,0.03207999964555105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,2047,0.042122667034467064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,2047,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,4095,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,2047,0.02438933402299881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,4095,0.03976000100374222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,4095,0.06322133541107178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,4095,0.03952533255020777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,8191,0.03975466638803482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,8191,0.08231999973456065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,8191,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,1,0.010490667074918747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,8191,0.05799466868241628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,1,0.010533332824707031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,1,0.0069440001000960665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,1,0.006784000123540561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,3,0.010474666953086853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,7,0.010234666367371878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,7,0.006927999978264173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,3,0.010405333091815313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,15,0.010538666198650995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,3,0.006778666749596596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,3,0.007087999954819679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,7,0.010538666198650995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,7,0.007040000210205714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,15,0.010341333225369453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,15,0.0069386667261521024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,15,0.0069866664707660675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,31,0.010602666685978571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,31,0.010474666953086853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,31,0.007061333085099856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,31,0.006800000245372455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,63,0.010298666854699453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,63,0.010277333358923594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,63,0.006965333595871925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,63,0.006981333096822103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,127,0.010602666685978571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,255,0.008629333227872849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,127,0.010469333579142889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,127,0.007482666522264481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,127,0.007333333293596904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,255,0.01848000039656957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,255,0.018351999421914417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,255,0.009621333330869675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,511,0.023232000569502514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,511,0.02502399931351344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,511,0.009397333487868309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,511,0.011770666887362799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,2047,0.03607466568549474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,1023,0.03322133421897888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,1023,0.03988266736268997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,1023,0.009722666814923286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,2047,0.02475200096766154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,1023,0.019866666446129482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,2047,0.05272533496220907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,2047,0.009765333185593287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,4095,0.03577066709597906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,4095,0.0678719977537791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,4095,0.009717333440979322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,4095,0.035391998787721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,8191,0.009519999846816063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,1,0.014565333724021912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,8191,0.05569600065549215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,1,0.014682666709025701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,8191,0.03558400024970373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,8191,0.08980799714724223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,1,0.006549333532651265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,1,0.006181333214044571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,3,0.014570667097965876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,3,0.014688000082969666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,3,0.006448000048597653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,3,0.006464000170429547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,7,0.014463999619086584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,15,0.01470400020480156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,7,0.014826666563749313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,7,0.006229333579540253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,7,0.00873066671192646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,15,0.014655999839305878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,15,0.006575999781489372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,15,0.0063786668082078295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,31,0.014794666320085526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,31,0.014773332824309668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,31,0.006271999950210254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,31,0.006186666587988536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,63,0.014677333335081736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,127,0.014581333845853806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,63,0.014831999937693277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,63,0.006351999938488007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,63,0.006474666918317477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,127,0.01463466634353002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,255,0.008645333349704742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,127,0.00696000022192796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,127,0.007055999711155891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,255,0.026186667382717133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,511,0.010090666512648264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,255,0.02672533442576726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,255,0.009621333330869675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,511,0.036303999523321785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,1023,0.04969066878159841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,511,0.0402399996916453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,511,0.015882667154073715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,1023,0.03585600107908249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,1023,0.010181333248813948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,1023,0.020154666155576706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,2047,0.03886399914820989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,2047,0.06746666630109151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,2047,0.010144000252087912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,4095,0.08955732981363933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,2047,0.02827200045188268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,4095,0.038319999972979225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,4095,0.010079999764760336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,4095,0.04622933268547058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,8191,0.03873066604137421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,1,0.022783999641736347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,1,0.022895999252796173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,8191,0.01003200002014637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,8191,0.13410133123397827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,1,0.00701333334048589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,8191,0.07782933115959167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,1,0.007087999954819679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,3,0.022853332261244457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,3,0.02293866624434789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,7,0.007082666580875714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,3,0.007354666789372762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,3,0.007125333572427432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,15,0.007071999832987785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,7,0.023050665855407715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,7,0.0230880007147789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,7,0.007061333085099856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,31,0.007189333438873291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,15,0.022970666488011677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,15,0.022965334355831146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,15,0.007167999943097432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,31,0.023018665611743927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,31,0.02295999974012375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,31,0.007173333317041397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,63,0.022624000906944275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,63,0.022885332504908245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,63,0.007007999966541926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,63,0.00701333334048589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,255,0.043247997760772705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,255,0.013445333888133367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,255,0.015365333606799444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,511,0.04317333300908407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,127,0.022709332406520844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,127,0.022789334257443745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,127,0.008053333188096682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,127,0.008282666405042013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,255,0.04273599882920583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,1023,0.06514666477839152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,511,0.05208533505598704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,511,0.013594667116800943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,1023,0.027717334528764088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,511,0.019754666835069656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,1023,0.04292800029118856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,2047,0.09141866366068523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,1023,0.01350933313369751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,2047,0.04562666515509287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,2047,0.013424000392357508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,2047,0.045034666856129967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,4095,0.04614933331807455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,4095,0.013455999394257864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,4095,0.1349386672178904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,4095,0.07720000048478444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,8191,0.045226668318112694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,1,0.007887999837597212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,1,0.007829333345095316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,8191,0.013568000247081121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,3,0.007962666451931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,3,0.00789866658548514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,1,0.007930666829148928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,8191,0.2253440022468567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,8191,0.14261333147684732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,1,0.00784533346692721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,3,0.008336000144481659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,3,0.007861333588759104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,7,0.007930666829148928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,7,0.007882666463653246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,7,0.007834666719039282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,7,0.00795199970404307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,15,0.007840000092983246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,15,0.007903999959429106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,15,0.007840000092983246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,15,0.008277333031098047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,31,0.008207999790708223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,31,0.008272000278035799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,31,0.008010666817426682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,31,0.0081386665503184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,63,0.008762666955590248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,63,0.008218666538596153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,63,0.00874133345981439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,63,0.008282666405042013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,127,0.009098666409651438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,127,0.00921066664159298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,127,0.009098666409651438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,127,0.009306666751702627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,255,0.008869333192706108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,255,0.009232000137368837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,255,0.010501333822806677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,255,0.010682666053374609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,511,0.009253333633144697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,511,0.0100853331387043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,511,0.009695999945203463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,511,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,1023,0.021909333765506744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,1023,0.02622399975856145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,1023,0.009503999724984169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,1023,0.012096000214417776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,2047,0.015066667149464289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,2047,0.015605332950750986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,2047,0.009589333087205887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,2047,0.012944000462690989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,4095,0.015935999651749928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,4095,0.01647466669480006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,4095,0.009493333597977957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,4095,0.01440000037352244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,8191,0.016447999825080235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,8191,0.00914666677514712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,8191,0.01950399950146675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,8191,0.01764800027012825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,16383,0.017488000293572743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,16383,0.02128533273935318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,16383,0.009530666594703993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,16383,0.020687999824682873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,32767,0.03298133363326391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,32767,0.020949333906173706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,32767,0.009472000102202097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,32767,0.029706666866938274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,1,0.007760000104705493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,1,0.007887999837597212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,1,0.006544000158707301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,1,0.006650666395823161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,3,0.007802666475375493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,65535,0.024010665714740753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,3,0.006602666651209195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,3,0.007861333588759104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,3,0.006618666773041089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,65535,0.04077333211898804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,7,0.007818666597207388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,65535,0.009637333452701569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,65535,0.03910933434963226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,15,0.008074666683872541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,7,0.008170666793982187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,31,0.008074666683872541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,31,0.008330666770537695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,7,0.00660800002515316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,7,0.006725333631038666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,15,0.007936000203092894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,15,0.006655999769767125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,15,0.006954666847983996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,31,0.008005333443482717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,31,0.006533333410819371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,63,0.008976000050703684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,63,0.008863999818762144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,63,0.006704000135262807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,63,0.006709333509206772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,127,0.009237333511312803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,255,0.00810666692753633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,127,0.00916800027092298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,255,0.009141333401203156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,127,0.007280000175038974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,127,0.007477333148320516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,255,0.009050666665037474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,255,0.00898133342464765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,511,0.008943999807039896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,511,0.010255999863147736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,511,0.008357333640257517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,511,0.009269333134094873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,1023,0.011994666109482447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,2047,0.014261333892742792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,1023,0.011994666109482447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,1023,0.008058666562040647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,1023,0.009488000224033991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,4095,0.015125333021084467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,4095,0.007887999837597212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,2047,0.01402666668097178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,2047,0.008090666805704435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,2047,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,4095,0.014170666535695394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,4095,0.012053333222866058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,8191,0.02181333303451538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,8191,0.017466666797796886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,8191,0.008245333408315977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,8191,0.016517333686351776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,16383,0.008234666660428047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,16383,0.01821333294113477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,16383,0.02589866767326991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,16383,0.019930666933457058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,32767,0.022074667116006214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,32767,0.038218667109807335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,32767,0.00808533343176047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,32767,0.027600000301996868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,1,0.008000000069538752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,1,0.00790933333337307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,1,0.005850666513045629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,1,0.006037333359320958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,3,0.007978666573762894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,3,0.007941333577036858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,65535,0.025727999707063038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,3,0.006778666749596596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,65535,0.0574186642964681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,3,0.00589866687854131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,7,0.00790933333337307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,65535,0.008303999900817871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,7,0.007925333455204964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,65535,0.04460800190766653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,7,0.006048000107208888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,7,0.006111999973654747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,31,0.008207999790708223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,31,0.006058666855096817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,15,0.008000000069538752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,15,0.00795199970404307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,15,0.005994666367769241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,15,0.005989333614706993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,31,0.00816000004609426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,31,0.005973333492875099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,63,0.008863999818762144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,63,0.00884799969693025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,63,0.005973333492875099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,63,0.005946666623155276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,127,0.009546666716535887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,127,0.00892800030608972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,127,0.006448000048597653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,127,0.006624000146985054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,255,0.009072000160813332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,255,0.008997333546479544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,255,0.007194666812817256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,255,0.007829333345095316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,1023,0.011722666521867117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,511,0.011488000551859537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,511,0.011418666690587997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,2047,0.016176000237464905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,511,0.007226666435599327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,511,0.008058666562040647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,1023,0.011567999919255575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,4095,0.016719999412695568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,1023,0.007194666812817256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,1023,0.008410666758815447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,4095,0.007322666545708974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,2047,0.018570666511853535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,2047,0.007391999786098798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,2047,0.0100426667680343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,4095,0.020666666328907013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,4095,0.011322667201360067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,8191,0.019839999576409657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,8191,0.02844800055027008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,8191,0.007541333635648091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,8191,0.015509333461523056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,16383,0.03868266691764196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,16383,0.02309866746266683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,16383,0.007424000029762586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,16383,0.020954666038354237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,32767,0.02870933214823405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,32767,0.06165333092212677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,32767,0.035455999275048576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,32767,0.007791999727487564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,1,0.007930666829148928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,1,0.007967999825874964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,1,0.005850666513045629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,1,0.006031999985376994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,3,0.007967999825874964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,3,0.007914666707317034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,65535,0.03489066660404205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,65535,0.09411733349164327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,3,0.006117333347598712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,3,0.00590933362642924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,7,0.007930666829148928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,65535,0.007733333234985669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,7,0.007914666707317034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,65535,0.053344001372655235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,7,0.0060159998635451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,7,0.006085333103934924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,15,0.008101333553592363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,31,0.008261333530147871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,15,0.00790933333337307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,15,0.005882666756709416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,15,0.0057386669019858045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,31,0.0081386665503184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,31,0.0061919999619325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,31,0.006026666611433029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,63,0.009008000294367472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,63,0.008874666566650072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,63,0.00589866687854131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,63,0.005904000252485275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,127,0.009008000294367472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,127,0.009290666629870733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,127,0.006319999694824219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,127,0.006442666674653689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,255,0.011472000430027643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,255,0.011333333949247995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,255,0.006874666859706243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,255,0.007338666667540868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,511,0.011877333124478659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,511,0.01201066623131434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,511,0.007269333427151044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,511,0.007696000238259633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,2047,0.019002666076024372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,1023,0.01598400001724561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,2047,0.0240639994541804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,2047,0.0074986666440963745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,1023,0.01714666684468587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,1023,0.007184000064929326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,1023,0.008986666798591614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,2047,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,4095,0.02319466571013133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,4095,0.03234133372704188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,4095,0.007327999919652939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,4095,0.01392000044385592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,8191,0.02731200059254964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,8191,0.045007998744646706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,8191,0.007573333258430163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,8191,0.022096000611782074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,16383,0.03370666752258936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,16383,0.0676693320274353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,16383,0.008090666805704435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,16383,0.030506665507952373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,32767,0.03490666548411051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,32767,0.007930666829148928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,32767,0.0913706620534261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,32767,0.05083733300367991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,1,0.011861333002646765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,1,0.011946666985750198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,1,0.008336000144481659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,1,0.008031999692320824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,3,0.011930666863918304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,3,0.012175999581813812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,3,0.008090666805704435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,65535,0.007797333101431529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,65535,0.034272000193595886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,65535,0.1386186679204305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,3,0.008186666915814081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,7,0.008063999935984612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,65535,0.08478933572769165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,7,0.012005332857370377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,15,0.011957333733638128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,7,0.012213333199421564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,7,0.008389333263039589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,15,0.012154666086037954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,15,0.00797333319981893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,15,0.008287999778985977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,31,0.012015999605258306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,31,0.011920000116030375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,31,0.008586666857202848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,31,0.008586666857202848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,63,0.012047999848922094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,127,0.00901333304742972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,63,0.011722666521867117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,127,0.008762666955590248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,63,0.008223999912540117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,127,0.011999999483426413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,63,0.00855466661353906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,127,0.01173866664369901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,255,0.02163200080394745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,255,0.02146133283774058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,511,0.029733332494894665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,255,0.010255999863147736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,511,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,255,0.011312000453472137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,511,0.02699200063943863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,511,0.018895999838908512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,1023,0.0391146664818128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,1023,0.04799466828505198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,1023,0.011493333925803503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,1023,0.028223998844623566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,2047,0.041759997606277466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,2047,0.036229332288106285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,2047,0.06257066627343495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,2047,0.011530666301647821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,4095,0.04152533411979675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,1,0.014869333555301031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,4095,0.08065600196520488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,4095,0.011525332927703857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,1,0.014837333311637243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,1,0.00721066693464915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,3,0.015029333531856537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,4095,0.05667733152707418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,1,0.007296000296870868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,3,0.014938666174809137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,3,0.007173333317041397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,7,0.007247999931375186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,3,0.007146666447321574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,7,0.015253332753976187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,7,0.01498666654030482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,7,0.007311999797821045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,31,0.014842666685581207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,15,0.014970666418472925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,31,0.014890667051076889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,15,0.014938666174809137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,15,0.007130666946371396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,15,0.007045333584149678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,31,0.007381333038210869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,63,0.00725333330531915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,31,0.00720000018676122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,63,0.014975999792416891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,63,0.014783999572197596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,63,0.007157333195209503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,127,0.014570667097965876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,127,0.01498666654030482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,255,0.009722666814923286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,127,0.008069333309928576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,127,0.007877333089709282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,255,0.027002667387326557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,255,0.02719466636578242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,255,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,511,0.036917333801587425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,511,0.04160533348719279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,511,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,511,0.019088000059127808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,1023,0.03658133248488108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,1023,0.024101334313551586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,1023,0.05115733544031779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,1023,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,2047,0.0393653338154157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,2047,0.06862399975458781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,2047,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,2047,0.03422400106986364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,1,0.023306667804718018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,4095,0.09097599983215332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,4095,0.03939199944337209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,1,0.007045333584149678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,1,0.007184000064929326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,4095,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,4095,0.05599466462930044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,1,0.02332266668478648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,3,0.023647998770078022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,7,0.023045333723227184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,3,0.007146666447321574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,3,0.02405333270629247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,15,0.023525332411130268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,7,0.023434666295846302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,3,0.007205333560705185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,7,0.007311999797821045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,7,0.007322666545708974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,15,0.02309866746266683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,15,0.007050666958093643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,15,0.0069759997228781385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,31,0.023530667026837666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,31,0.023381332556406658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,31,0.007184000064929326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,31,0.007269333427151044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,63,0.023045333723227184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,63,0.022976001103719074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,63,0.007141333073377609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,63,0.0068693334857622785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,127,0.023567999402681988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,127,0.023205332458019257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,127,0.008229333286484083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,127,0.008458666503429413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,255,0.04324266811211904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,511,0.052330667773882546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,255,0.043050666650136314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,255,0.013765333841244379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,511,0.0199946661790212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,255,0.015717333803574245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,511,0.04378133515516917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,511,0.013776000589132309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,1023,0.04330666859944662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,1023,0.06599999964237213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,1023,0.013861333330472311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,1023,0.028757333755493164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,2047,0.04599999884764353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,2047,0.09180800120035808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,2047,0.013845333208640417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,2047,0.04599999884764353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,4095,0.04578666885693868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,4095,0.13622933626174927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,4095,0.013823999712864557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,1,0.040031999349594116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,1,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,4095,0.07851199805736542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,1,0.03979733337958654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,1,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,3,0.039690665900707245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,7,0.03967999915281931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,3,0.03962666789690653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,3,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,3,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,7,0.039647998909155525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,7,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,7,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,15,0.03966933240493139
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,15,0.03959999978542328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,15,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,15,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,31,0.03972266614437103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,31,0.039349332451820374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,31,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,31,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,63,0.03923733284076055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,63,0.039503999054431915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,63,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,63,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,127,0.03909866760174433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,127,0.03920000046491623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,255,0.04084266722202301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,127,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,127,0.012981332838535309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,255,0.04107200105985006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,255,0.012752000242471695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,255,0.01659199967980385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,511,0.04119466741879781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,511,0.0539626677831014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,511,0.012784000486135483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,511,0.025199999411900837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,1023,0.04131199916203817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,1023,0.07751999795436859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,1023,0.012634667257467905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,1023,0.042117332418759666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,2047,0.044122666120529175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,2047,0.1250986655553182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,2047,0.012869333227475485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,2047,0.0740479975938797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,4095,0.04411733150482178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,4095,0.012576000144084295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,4095,0.2133493423461914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,float16,1,0.017642666896184284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,4095,0.13778133193651834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,1,0.017738666385412216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,fp8,1,0.008703999842206636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,1,0.008656000097592672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,float16,3,0.017530667285124462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,3,0.017637333522240322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,fp8,3,0.008757333581646284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,float16,7,0.017690667261679966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,3,0.008805333326260248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,7,0.017642666896184284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,fp8,7,0.00867733359336853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,7,0.008592000231146812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,float16,15,0.01756799966096878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,15,0.017738666385412216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,fp8,31,0.008586666857202848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,fp8,15,0.008863999818762144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,float16,63,0.017370666066805523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,15,0.008746666833758354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,float16,31,0.01758933315674464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,31,0.017759999881188076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,31,0.008650666723648706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,63,0.017557332913080852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,fp8,63,0.00873066671192646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,127,0.009594666461149851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,63,0.008767999708652496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,float16,127,0.01757866640885671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,127,0.017653333644072216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,fp8,127,0.009658666948477427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,float16,255,0.032458665470282234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,float16,511,0.044250667095184326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,255,0.03233599911133448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,fp8,255,0.016970666746298473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,255,0.018992000569899876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,511,0.04920533299446106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,float16,1023,0.043375998735427856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,fp8,511,0.017221332838137943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,511,0.027978666126728058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,1023,0.06148266792297363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,fp8,1023,0.017269333203633625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,float16,1,0.023984000086784363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,1023,0.03509333233038584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,1,0.02359466751416524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,fp8,1,0.007871999715765318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,1,0.0081386665503184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,float16,3,0.024127999941507976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,3,0.02390933285156886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,fp8,3,0.007957333077987036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,3,0.007946666950980822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,float16,7,0.023951999843120575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,7,0.023872000475724537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,fp8,7,0.008165333420038223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,7,0.007967999825874964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,15,0.007871999715765318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,float16,15,0.023936000963052113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,15,0.024005333582560223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,fp8,15,0.007882666463653246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,31,0.007920000081261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,float16,31,0.023599999646345775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,31,0.024186665813128155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,63,0.02362666775782903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,fp8,31,0.008282666405042013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,float16,63,0.02342933416366577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,fp8,63,0.007887999837597212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,63,0.007877333089709282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,fp8,127,0.009466666728258133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,float16,127,0.023333333432674408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,127,0.02367466688156128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,127,0.00949866697192192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,float16,255,0.0439573327700297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,255,0.043749332427978516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,fp8,255,0.01589866727590561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,255,0.018511999398469925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,float16,511,0.04390400151411692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,511,0.05409066875775655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,fp8,511,0.015962666521469753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,511,0.023658665517965954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,float16,1023,0.044026667873064675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,float16,1,0.04070399949947993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,1023,0.06788266698519389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,fp8,1023,0.015791999797026317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,1023,0.035162667433420815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,1,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,1,0.04067199925581614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,fp8,1,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,float16,3,0.04074133435885111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,float16,7,0.04012800008058548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,3,0.040133332212766014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,fp8,3,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,3,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,7,0.04043733328580856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,fp8,7,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,7,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,float16,15,0.0400693342089653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,15,0.0402399996916453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,fp8,15,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,15,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,float16,63,0.03972800076007843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,float16,31,0.04030933231115341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,31,0.0401653324564298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,fp8,31,0.01138666644692421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,31,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,63,0.03962666789690653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,fp8,63,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,63,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,float16,127,0.03979733337958654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,127,0.03949866692225138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,fp8,127,0.013327999661366144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,127,0.013290667285521826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,float16,255,0.04191466669241587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,255,0.04153066625197729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,fp8,255,0.013130666067202887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,255,0.017269333203633625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,float16,511,0.04274666806062063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,511,0.05606933434804281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,fp8,511,0.013194666554530462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,511,0.026165333886941273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,float16,1023,0.04251733422279358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,1023,0.07850666840871175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,fp8,1023,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,1023,0.0431573341290156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,float16,1,0.07317866881688435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,1,0.07217599948247273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,fp8,1,0.01828266680240631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,1,0.01848000039656957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,float16,7,0.07316266496976216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,float16,3,0.07237866520881653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,3,0.07262933254241943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,fp8,3,0.01850133389234543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,3,0.01846933364868164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,15,0.07222933570543925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,7,0.07295999924341838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,fp8,7,0.018351999421914417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,float16,31,0.07201600074768066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,7,0.01844266677896182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,float16,15,0.07329600056012471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,fp8,15,0.01871466636657715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,15,0.018511999398469925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,31,0.07252799967924754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,fp8,31,0.018618666877349217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,63,0.018570666511853535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,31,0.018415999909241993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,float16,63,0.0713973343372345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,63,0.07227733234564464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,127,0.07160000006357829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,fp8,63,0.018458666900793713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,float16,127,0.07211199899514516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,fp8,255,0.02162666618824005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,fp8,127,0.02165333429972331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,127,0.021498667697111767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,float16,255,0.07705600063006084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,255,0.07760000228881836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,255,0.030506665507952373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,float16,511,0.07706666489442189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,511,0.09672000010808308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,fp8,511,0.021722666919231415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,511,0.046896000703175865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,float16,1023,0.07657599945863088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,1023,0.14341333508491516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,fp8,1023,0.021888000269730885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,1023,0.07830933233102162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,float16,1,0.02938666691382726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,1,0.029578665892283123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,fp8,1,0.014074667046467463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,1,0.014175999909639359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,float16,3,0.02956799914439519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,fp8,3,0.014069333672523499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,3,0.02937600016593933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,3,0.014175999909639359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,float16,7,0.02939733366171519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,7,0.02937600016593933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,fp8,7,0.014181333283583323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,7,0.014335999886194864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,float16,15,0.029194665451844532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,15,0.02942933390537898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,fp8,15,0.014271999398867289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,15,0.014256000518798828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,float16,31,0.02951466788848241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,31,0.029440000653266907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,float16,63,0.029311999678611755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,fp8,31,0.014325333138306936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,31,0.013951999445756277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,63,0.028970666229724884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,fp8,63,0.014138666292031607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,63,0.014218666901191076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,float16,127,0.02906133234500885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,127,0.029285334050655365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,fp8,127,0.01617066686352094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,127,0.015909332782030106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,float16,255,0.05449066559473673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,255,0.05417599777380625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,fp8,255,0.024341332415739696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,511,0.0668213317791621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,255,0.02749866743882497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,511,0.035973332822322845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,float16,511,0.05529599885145823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,fp8,511,0.024186665813128155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,1,0.012773333738247553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,float16,1,0.04204266766707102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,1,0.04167466859022776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,fp8,1,0.012367999802033106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,float16,3,0.041450666884581246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,3,0.0415786678592364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,fp8,3,0.012698666503032049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,3,0.012469333906968435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,float16,7,0.04203199843565623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,7,0.04223999877770742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,float16,15,0.041434665520985924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,fp8,15,0.012624000509579977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,fp8,7,0.012682666381200155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,7,0.012373333175977072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,float16,31,0.04201066493988037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,15,0.04176533222198486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,15,0.01251199965675672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,fp8,31,0.012448000411192576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,31,0.042266666889190674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,31,0.012533333152532578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,float16,63,0.04067733387152354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,63,0.04083733260631561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,fp8,63,0.012618667135636011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,63,0.012485332787036896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,fp8,127,0.015354666858911514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,float16,127,0.04074666649103165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,127,0.04073066761096319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,127,0.015402667224407196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,float16,255,0.044906665881474815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,255,0.04358399907747904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,fp8,255,0.01543466622630755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,255,0.02051199972629547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,511,0.0322080006202062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,float16,511,0.04423999786376953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,511,0.057722667853037514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,fp8,511,0.015061333775520325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,1,0.018730666488409042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,float16,1,0.07348266740640004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,1,0.07322666545708974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,fp8,1,0.019002666076024372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,float16,3,0.07374399900436401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,3,0.0737013320128123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,fp8,3,0.018687999496857326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,3,0.018746666610240936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,float16,7,0.073253333568573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,7,0.07316266496976216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,fp8,7,0.019152000546455383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,7,0.018965333700180054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,float16,15,0.07356800138950348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,15,0.01878400022784869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,15,0.07356266677379608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,fp8,15,0.01882133384545644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,float16,31,0.0734986662864685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,31,0.07436800003051758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,fp8,31,0.019029332945744198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,31,0.019061333189407986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,float16,63,0.07238933444023132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,63,0.07274133463700612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,fp8,63,0.018735999862353008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,127,0.07448533177375793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,63,0.018778666853904724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,float16,127,0.07477866609891255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,fp8,127,0.022661333282788593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,127,0.02233600119749705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,float16,255,0.07980800171693166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,fp8,255,0.022592000663280487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,255,0.07901866734027863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,255,0.03160000095764796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,float16,511,0.08040000001589458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,511,0.09981866677602132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,fp8,511,0.02236266682545344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,511,0.04881600042184194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,float16,1,0.13589866956075033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,1,0.13487999637921652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,fp8,1,0.0323840007185936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,1,0.03293866664171219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,float16,3,0.13539733489354452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,3,0.03271466741959254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,3,0.13435733318328857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,fp8,3,0.032431999842325844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,fp8,7,0.03291733314593633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,float16,7,0.13482133547465006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,7,0.1349546710650126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,float16,15,0.13595199584960938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,7,0.03259200106064478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,15,0.13452266653378805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,fp8,15,0.03256533294916153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,15,0.0330079992612203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,float16,31,0.13493333260218301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,fp8,31,0.032746667663256325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,31,0.13495999574661255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,31,0.03262399882078171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,float16,63,0.13532267014185587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,63,0.1365653375784556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,fp8,63,0.03293866664171219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,127,0.135535995165507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,63,0.03234666585922241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,float16,127,0.13577066858609518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,fp8,127,0.04141333450873693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,127,0.04170133173465729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,fp8,255,0.04222933451334635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,255,0.14320533474286398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,float16,255,0.14342400431632996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,255,0.056885331869125366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,1,0.008714666590094566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,float16,511,0.1442400018374125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,1,0.007936000203092894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,1,0.008602666358153025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,511,0.18145600954691568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,1,0.007717333113153775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,fp8,511,0.04161600023508072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,3,0.008762666955590248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,511,0.08700799942016602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,7,0.008405333384871483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,3,0.008725333337982496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,3,0.007829333345095316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,3,0.007850666840871176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,7,0.00892800030608972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,15,0.007882666463653246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,7,0.00873066671192646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,7,0.008527999743819237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,15,0.008810666700204214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,15,0.008890666688481966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,15,0.008432000254591307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,31,0.009397333487868309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,31,0.009594666461149851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,127,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,31,0.007733333234985669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,127,0.011322667201360067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,127,0.008986666798591614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,31,0.008154666672150293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,127,0.008682666967312494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,63,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,63,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,63,0.008362666393319765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,63,0.008485333373149237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,255,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,255,0.011285333583752314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,255,0.009546666716535887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,255,0.010154666379094124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,511,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,511,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,511,0.009621333330869675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,511,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,1023,0.01312000056107839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,1023,0.01339200014869372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,2047,0.009418666362762451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,1023,0.009301333377758661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,1023,0.011365332951148352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,2047,0.01525866612792015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,4095,0.009242666885256767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,4095,0.014645333091417948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,2047,0.01616000011563301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,2047,0.01259200026591619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,4095,0.015439999600251516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,4095,0.01637866720557213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,8191,0.019551999866962433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,8191,0.024527999262015026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,8191,0.009488000224033991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,16383,0.010037333394090334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,8191,0.020629333953062694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,16383,0.02062400057911873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,16383,0.029279999434947968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,16383,0.024688000480333965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,32767,0.024885334074497223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,32767,0.009493333597977957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,32767,0.04524266719818115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,32767,0.039093332986036934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,1,0.008661333471536636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,1,0.00877333308259646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,1,0.006815999746322632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,1,0.006655999769767125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,3,0.008714666590094566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,3,0.008826666822036108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,3,0.006650666395823161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,3,0.006671999891599019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,7,0.008938666433095932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,7,0.008752000207702318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,7,0.006671999891599019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,65535,0.029232000311215717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,7,0.007173333317041397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,65535,0.009930666536092758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,15,0.008810666700204214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,65535,0.06911999980608623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,65535,0.07341333230336507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,15,0.008943999807039896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,15,0.006677333265542984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,15,0.007040000210205714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,31,0.009429333110650381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,31,0.009568000212311745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,31,0.006762666627764702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,31,0.006549333532651265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,63,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,63,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,63,0.007061333085099856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,63,0.006821333120266597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,127,0.012181332955757776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,127,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,127,0.007231999809543292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,127,0.007653333246707916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,255,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,255,0.011674666156371435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,255,0.008143999924262365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,255,0.00897066667675972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,511,0.011525332927703857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,511,0.011434666812419891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,511,0.008127999802430471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,1023,0.009418666362762451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,511,0.009072000160813332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,1023,0.012133333832025528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,1023,0.011882666498422623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,1023,0.008063999935984612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,2047,0.016106666376193363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,2047,0.019007999449968338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,2047,0.0116799995303154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,2047,0.008458666503429413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,4095,0.016927999754746754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,4095,0.020768000433842342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,8191,0.008453333129485449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,4095,0.008597333605090777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,4095,0.013909333695967993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,8191,0.020234666764736176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,8191,0.029088000456492107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,8191,0.01905599981546402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,16383,0.023423999547958374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,16383,0.008320000022649765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,16383,0.03998400022586187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,16383,0.02515200028816859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,32767,0.02917333443959554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,32767,0.06233599781990051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,32767,0.008698666468262672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,32767,0.044682666659355164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,1,0.008767999708652496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,1,0.00867733359336853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,1,0.006069333602984746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,1,0.005930666501323382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,3,0.008816000074148178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,3,0.008682666967312494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,65535,0.00902399979531765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,3,0.006239999706546466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,65535,0.03547733277082443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,65535,0.09521599610646565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,3,0.005989333614706993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,7,0.006042666733264923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,65535,0.0668213317791621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,7,0.008821333448092142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,7,0.009066666786869368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,31,0.009519999846816063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,7,0.006298666819930077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,15,0.008826666822036108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,15,0.008858666444818178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,15,0.00590933362642924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,63,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,15,0.006159999718268712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,31,0.009637333452701569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,31,0.005930666501323382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,31,0.006224000205596288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,63,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,63,0.006079999729990959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,127,0.006506666541099548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,63,0.005973333492875099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,255,0.011839999506870905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,127,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,127,0.011488000551859537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,127,0.0063733334342638654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,255,0.011642667154471079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,255,0.007407999907930692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,255,0.007685333490371704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,511,0.011690666278203329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,511,0.011968000481526056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,511,0.007520000139872233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,511,0.00797333319981893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,1023,0.01617066686352094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,1023,0.01739199956258138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,1023,0.007296000296870868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,1023,0.009066666786869368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,2047,0.019066666563351948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,2047,0.02442666639884313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,2047,0.007653333246707916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,2047,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,4095,0.02407466620206833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,4095,0.032085334261258446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,8191,0.04443199932575226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,8191,0.007733333234985669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,4095,0.007717333113153775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,4095,0.014218666901191076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,8191,0.027471999327341717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,16383,0.033376000821590424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,8191,0.02345066765944163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,16383,0.06829333305358887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,16383,0.007882666463653246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,16383,0.03148799886306127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,32767,0.03469866762558619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,32767,0.008154666672150293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,32767,0.09236799677213033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,32767,0.054058666030565895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,1,0.010117333382368088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,1,0.010202666744589806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,1,0.005968000118931134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,1,0.005946666623155276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,3,0.010181333248813948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,3,0.010048000141978264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,3,0.005957333371043205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,3,0.006282666698098183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,65535,0.033946665624777474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,7,0.0100426667680343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,7,0.010101333260536194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,7,0.006218666831652324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,7,0.0058186668902635574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,65535,0.13928000132242838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,15,0.010191999996701876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,65535,0.08524800340334575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,65535,0.008026666939258575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,15,0.009877333417534828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,15,0.006106666599710782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,15,0.0058666666348775225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,31,0.010026666646202406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,31,0.010117333382368088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,31,0.00620266670982043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,31,0.006117333347598712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,127,0.010048000141978264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,63,0.009898666913310686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,63,0.010037333394090334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,255,0.011936000237862269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,63,0.006325333068768184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,63,0.005797333394487699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,127,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,127,0.006224000205596288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,511,0.016735999534527462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,127,0.006309333567818006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,511,0.008117333054542542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,255,0.012106666962305704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,255,0.007082666580875714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,255,0.007567999884486198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,511,0.016293333222468693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,511,0.007344000041484833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,1023,0.021141332884629566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,1023,0.024256000916163128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,1023,0.007690666864315669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,1023,0.010399999717871347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,4095,0.03364266703526179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,4095,0.051882664362589516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,2047,0.026848000784715016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,2047,0.03515200068553289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,2047,0.00814933329820633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,2047,0.01603200038274129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,4095,0.008240000034372011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,4095,0.020554666717847187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,8191,0.03349333256483078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,8191,0.06654933094978333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,8191,0.007925333455204964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,16383,0.04601066807905833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,8191,0.02923733244339625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,16383,0.03352533280849457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,16383,0.09021866321563721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,32767,0.13302399714787802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,16383,0.008250666782259941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,32767,0.07855999966462453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,32767,0.03408000121514002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,32767,0.008309333274761835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,65535,0.0336053321758906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,65535,0.22512533267339072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,65535,0.008303999900817871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,float16,1,0.05273599922657013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,float16,1,0.052095999320348106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,fp8,1,0.01988799994190534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,65535,0.14272532860438028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,fp8,1,0.01969066634774208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,float16,3,0.052629331747690834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,float16,3,0.05202666421731313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,fp8,3,0.020015999674797058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,fp8,3,0.019893333315849304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,float16,7,0.05186133086681366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,float16,7,0.05193066596984863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,fp8,7,0.019717333217461903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,float16,15,0.05208000044027964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,fp8,7,0.02013333390156428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,float16,15,0.052928000688552856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,fp8,15,0.0198186660806338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,fp8,15,0.019674666225910187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,float16,31,0.051957334081331887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,float16,63,0.05211733281612396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,float16,31,0.052341332038243614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,fp8,31,0.019823999454577763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,fp8,31,0.019920000185569126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,float16,127,0.05156800150871277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,float16,127,0.05188799897829691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,float16,63,0.05190399785836538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,fp8,63,0.019653332730134327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,fp8,127,0.02327999969323476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,fp8,63,0.019493332753578823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,fp8,127,0.023317334552605946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,float16,255,0.05522133409976959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,float16,255,0.05453333258628845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,fp8,255,0.030554667115211487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,float16,1,0.07634133100509644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,fp8,255,0.023221333821614582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,float16,1,0.07599466542402904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,fp8,1,0.02161066730817159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,fp8,1,0.021669333179791767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,float16,3,0.0759200006723404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,float16,3,0.07617599765459697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,fp8,3,0.02203733225663503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,fp8,3,0.021888000269730885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,float16,7,0.07666133344173431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,float16,7,0.07603733241558075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,fp8,7,0.02162666618824005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,fp8,7,0.021717332303524017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,float16,15,0.07598400115966797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,float16,15,0.0761599987745285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,fp8,15,0.021946666141351063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,float16,31,0.07608533402283986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,fp8,31,0.0216799999276797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,fp8,15,0.02186133215824763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,float16,31,0.07684266567230225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,float16,63,0.07441066702206929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,fp8,31,0.021701333423455555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,float16,63,0.07459733386834462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,fp8,63,0.021850667893886566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,float16,127,0.07659199833869934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,fp8,63,0.021536000072956085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,float16,127,0.07670400043328603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,fp8,127,0.027109332382678986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,fp8,127,0.026661333938439686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,float16,255,0.08231466511885326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,float16,255,0.08195200065771739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,fp8,255,0.04111466556787491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,float16,1,0.1365600029627482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,fp8,255,0.02754666656255722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,fp8,1,0.034048000971476235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,float16,3,0.13710400462150574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,float16,1,0.13699733217557272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,fp8,1,0.03321066747109095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,fp8,3,0.03333866596221924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,float16,3,0.13685866196950278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,fp8,3,0.033413333197434746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,float16,7,0.1373599966367086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,float16,7,0.13867732882499695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,fp8,7,0.033200000723203026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,fp8,7,0.033610666791598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,float16,15,0.1367039978504181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,float16,31,0.13911466797192892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,float16,15,0.13728533188501993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,float16,31,0.1376159985860189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,fp8,15,0.03373866776625315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,fp8,15,0.03420799970626831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,fp8,31,0.03357866654793421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,fp8,63,0.03294399877389272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,float16,63,0.137882669766744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,fp8,31,0.03324266771475474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,float16,127,0.14059199889500937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,float16,63,0.1397546629110972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,fp8,63,0.033471999069054924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,float16,127,0.14019200205802917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,fp8,127,0.04492799937725067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,fp8,255,0.04499199986457825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,fp8,127,0.04470933477083842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,float16,255,0.14827199776967367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,float16,255,0.14808533589045206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,fp8,255,0.05961599946022034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,float16,1,0.2618880073229472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,float16,1,0.261845330397288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,fp8,1,0.06900799771149953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,fp8,1,0.06897066533565521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,float16,3,0.2610773245493571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,float16,3,0.261957327524821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,fp8,7,0.06881066660086314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,fp8,3,0.06835733354091644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,float16,15,0.26157333453496295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,fp8,3,0.0688266654809316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,float16,7,0.26159467299779254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,float16,7,0.2619679967562358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,fp8,7,0.06876266499360402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,float16,15,0.2625439961751302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,fp8,15,0.06888533135255177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,fp8,15,0.06832533578077953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,float16,31,0.2653866608937581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,float16,31,0.26693334182103473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,fp8,31,0.06822933256626129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,fp8,63,0.068122665087382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,fp8,31,0.06878933310508728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,float16,63,0.26595733563105267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,float16,63,0.26683199405670166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,float16,127,0.26387200752894086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,fp8,63,0.0687253326177597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,float16,127,0.2629759907722473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,fp8,127,0.0790880024433136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,fp8,127,0.07905066510041554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,fp8,255,0.10762133200963338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,float16,255,0.276309331258138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,float16,255,0.2759360074996948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,fp8,255,0.07971199850241344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,float16,1,0.09678399562835693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,float16,1,0.0957973301410675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,fp8,1,0.035605333745479584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,fp8,1,0.03703466554482778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,float16,3,0.09634133179982503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,float16,3,0.09554133812586467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,fp8,3,0.03735466549793879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,float16,7,0.09658132990201314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,fp8,3,0.03647999962170919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,float16,7,0.09587732950846355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,fp8,7,0.03586666782697042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,fp8,7,0.03722133239110311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,float16,15,0.09655466675758362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,fp8,15,0.03661333272854487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,float16,15,0.09645332892735799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,fp8,15,0.036943999429543815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,float16,31,0.09648533662160237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,float16,31,0.09635200103123982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,fp8,31,0.036943999429543815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,fp8,31,0.035674666364987694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,float16,63,0.09657599528630574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,float16,63,0.09674666325251262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,fp8,63,0.0359199990828832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,fp8,63,0.03689600030581156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,float16,127,0.09739200274149577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,float16,127,0.0974133312702179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,fp8,127,0.04629333317279816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,fp8,127,0.046069333950678505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,fp8,1,0.04241600135962168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,fp8,1,0.043680002291997276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,float16,1,0.1425333321094513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,float16,1,0.1422826647758484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,float16,3,0.1421013375123342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,float16,3,0.14171199997266135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,fp8,3,0.04330666859944662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,fp8,3,0.043285335103670754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,float16,7,0.14235732952753702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,float16,7,0.14226667086283365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,fp8,7,0.043285335103670754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,float16,15,0.14276267091433206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,float16,15,0.14193600416183472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,fp8,7,0.04375466704368591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,fp8,15,0.04345599810282389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,fp8,15,0.043050666650136314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,float16,31,0.14408000310262045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,float16,31,0.14402666687965393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,fp8,31,0.04334400097529093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,fp8,31,0.042223999897638954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,float16,63,0.1437333325544993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,float16,63,0.14479999740918478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,fp8,63,0.0432533323764801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,fp8,63,0.04203199843565623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,float16,127,0.1464853286743164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,float16,127,0.14611732959747314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,fp8,127,0.05909333129723867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,fp8,127,0.05890133480230967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,float16,1,0.2659306724866231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,float16,1,0.2651146650314331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,fp8,1,0.07116266588370006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,fp8,1,0.07177599767843883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,float16,3,0.26689600944519043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,float16,3,0.2659413417180379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,fp8,3,0.07150933146476746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,fp8,3,0.07166400055090587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,float16,7,0.26579733689626056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,float16,7,0.2660213311513265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,fp8,7,0.07153066496054332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,fp8,7,0.07148266832033794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,float16,15,0.2688693404197693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,float16,15,0.2680746714274089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,fp8,15,0.07152000069618225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,fp8,15,0.07101333141326904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,float16,31,0.2712106704711914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,fp8,31,0.07141866783301036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,float16,31,0.2723039984703064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,fp8,31,0.07108266651630402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,float16,63,0.2713013291358948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,float16,63,0.2730826735496521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,fp8,63,0.07109866539637248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,float16,127,0.272271990776062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,float16,127,0.273637334505717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,fp8,127,0.08433600266774495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,fp8,127,0.08400000135103862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,float16,1,0.5121920108795166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,fp8,1,0.13243732849756876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,fp8,1,0.13184000054995218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,float16,3,0.5177973508834839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,fp8,3,0.13185066978136697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,float16,3,0.5163360039393107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,fp8,3,0.1318773329257965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,float16,7,0.5177119970321655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,float16,7,0.5176213184992472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,fp8,7,0.13216533263524374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,float16,15,0.5226613283157349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,fp8,15,0.13215999801953635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,float16,31,0.5273493528366089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,float16,31,0.5279786586761475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,fp8,31,0.13194666306177774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,fp8,63,0.07081600030263265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,fp8,31,0.13186132907867432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,float16,1,0.515999992688497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,float16,63,0.5245653390884399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,float16,63,0.5251040061314901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,fp8,63,0.13206400473912558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,fp8,63,0.13136000434557596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,1,0.013936000565687815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,float16,127,0.5193706750869751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,1,0.013888000200192133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,1,0.00789866658548514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,1,0.007957333077987036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,fp8,127,0.15184533596038818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,float16,15,0.5248533487319946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,fp8,127,0.15275200208028158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,3,0.007829333345095316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,3,0.013893333574136099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,7,0.01413333291808764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,3,0.013872000078360239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,fp8,7,0.13194666306177774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,3,0.008170666793982187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,fp8,15,0.13157332936922708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,15,0.007967999825874964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,7,0.014111999422311783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,15,0.007834666719039282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,31,0.01551466683546702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,7,0.007840000092983246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,7,0.007829333345095316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,15,0.014282666146755219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,15,0.01414399966597557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,31,0.015354666858911514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,31,0.007701333612203598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,31,0.00808533343176047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,63,0.018533332894245785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,63,0.01852799952030182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,63,0.00797333319981893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,63,0.007834666719039282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,127,0.01870399961868922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,float16,127,0.5156586567560831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,127,0.018581333259741466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,127,0.00892800030608972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,127,0.009253333633144697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,255,0.01874133323629697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,255,0.019013332823912304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,255,0.009370666618148485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,255,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,511,0.012885333349307379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,511,0.012997332960367203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,511,0.00972800018886725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,511,0.010629333555698395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,1023,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,1023,0.013466666142145792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,2047,0.010015999898314476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,1023,0.009349333122372627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,1023,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,4095,0.019215999792019527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,2047,0.01850133389234543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,2047,0.02107200026512146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,2047,0.01575999955336253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,4095,0.023413332800070446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,4095,0.00984533317387104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,4095,0.016800000021855038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,8191,0.022965334355831146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,8191,0.03342933456103007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,8191,0.01027199998497963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,16383,0.00996800015370051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,8191,0.024447999894618988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,16383,0.02683199942111969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,16383,0.04730666677157084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,16383,0.04330133398373922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,32767,0.03347733368476232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,1,0.013850666582584381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,32767,0.009994666402538618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,32767,0.07597333192825317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,1,0.01381333296497663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,1,0.006751999879876773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,32767,0.061621333161989846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,1,0.006714666883150737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,3,0.013909333695967993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,3,0.013861333330472311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,3,0.006831999868154526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,3,0.006597333277265231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,7,0.014111999422311783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,7,0.013962666193644205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,7,0.006597333277265231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,7,0.006917333230376244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,15,0.014106666048367819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,15,0.014394666999578476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,15,0.006831999868154526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,15,0.006506666541099548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,31,0.01525866612792015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,31,0.015210667004187902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,31,0.006613333399097125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,31,0.006789333497484525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,127,0.01889066646496455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,63,0.018559999763965607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,63,0.018778666853904724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,63,0.006911999856432279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,63,0.006495999793211619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,127,0.01874133323629697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,127,0.007471999774376552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,127,0.007429333403706551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,255,0.01166933278242747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,255,0.011589333415031433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,255,0.008272000278035799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,1023,0.017301333447297413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,255,0.008805333326260248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,511,0.01139733319481214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,511,0.011749333391586939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,2047,0.024069334069887798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,511,0.008112000301480293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,511,0.008922666932145754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,1023,0.0163680004576842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,1023,0.008650666723648706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,1023,0.010570666442314783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,2047,0.019413333386182785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,2047,0.00879466657837232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,2047,0.013274667163689932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,4095,0.023797333240509033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,4095,0.0324799989660581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,4095,0.0085333331177632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,4095,0.017786666750907898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,8191,0.02792000025510788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,8191,0.04622933268547058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,8191,0.00867733359336853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,8191,0.027834666272004444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,16383,0.03414933383464813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,16383,0.06984533369541168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,16383,0.009061333412925402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,16383,0.03955200066169103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,32767,0.035258665680885315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,1,0.009877333417534828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,1,0.00996800015370051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,32767,0.0935093363126119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,1,0.005935999875267346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,32767,0.009002666920423508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,1,0.006048000107208888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,32767,0.0658186674118042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,3,0.010533332824707031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,3,0.009877333417534828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,3,0.00702400008837382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,3,0.005882666756709416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,7,0.010277333358923594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,7,0.010079999764760336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,7,0.007055999711155891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,7,0.007381333038210869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,15,0.010293333480755487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,15,0.009925333162148794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,15,0.00596266674498717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,15,0.0059199997534354525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,31,0.010293333480755487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,31,0.010250666489203772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,31,0.006095999851822853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,31,0.005973333492875099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,63,0.00985599992175897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,63,0.009882666791478792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,63,0.0058453331391016645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,63,0.006122666721542676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,127,0.01032533310353756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,127,0.01009599988659223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,127,0.006458666796485583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,127,0.006266666576266289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,255,0.01191466674208641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,511,0.007365333537260692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,255,0.011893333246310553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,255,0.007280000175038974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,255,0.007930666829148928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,511,0.016656000167131424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,511,0.017050666113694508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,511,0.008469333251317343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,1023,0.021375998854637146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,1023,0.024400000770886738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,1023,0.008016000191370646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,1023,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,4095,0.033802665770053864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,2047,0.027280000348885853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,2047,0.03552533437808355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,2047,0.00820266641676426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,8191,0.06692799925804138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,2047,0.016421332955360413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,4095,0.0526506652434667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,4095,0.008389333263039589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,4095,0.02160000056028366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,8191,0.03401600072781245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,8191,0.008293333152929941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,8191,0.02957333376010259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,16383,0.034101332227389015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,16383,0.08887466788291931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,16383,0.008383999889095625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,16383,0.047237331668535866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,32767,0.03401066611210505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,1,0.010351999973257383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,32767,0.13365866740544638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,32767,0.008207999790708223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,1,0.0102186668664217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,32767,0.08021333316961925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,1,0.006239999706546466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,1,0.006037333359320958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,3,0.010368000095089277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,3,0.01055466632048289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,3,0.005978666866819064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,3,0.006042666733264923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,15,0.006122666721542676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,7,0.010319999729593595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,7,0.010351999973257383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,7,0.006282666698098183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,31,0.010277333358923594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,7,0.006154666965206464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,15,0.010373333469033241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,15,0.010645333677530289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,15,0.006159999718268712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,31,0.010255999863147736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,31,0.006074666976928711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,31,0.006021333237489064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,63,0.010255999863147736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,63,0.010234666367371878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,63,0.006031999985376994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,63,0.006293333445986112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,127,0.010415999839703241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,127,0.010341333225369453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,127,0.0063146669417619705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,127,0.006442666674653689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,255,0.017984000345071156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,255,0.017818666994571686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,255,0.007311999797821045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,255,0.008293333152929941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,511,0.022661333282788593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,511,0.024688000480333965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,511,0.008266666904091835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,511,0.009466666728258133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,1023,0.03272533416748047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,2047,0.00921066664159298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,1023,0.03941866755485535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,1023,0.00850133349498113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,1023,0.016282666474580765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,2047,0.03484266748030981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,4095,0.0278613343834877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,2047,0.05075199902057648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,2047,0.019695999721686046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,4095,0.034703999757766724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,4095,0.06625066697597504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,4095,0.008527999743819237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,8191,0.03505599995454153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,8191,0.08844266335169475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,8191,0.008389333263039589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,8191,0.044981335600217186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,16383,0.03465600063403448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,16383,0.13235732913017273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,16383,0.008602666358153025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,16383,0.0767680009206136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,1,0.013616000612576803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,32767,0.2219840089480082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,32767,0.008682666967312494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,32767,0.03530666728814443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,1,0.009343999748428663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,3,0.014165333161751429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,3,0.009258666386206945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,32767,0.14065600434939066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,7,0.014538666854302088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,7,0.009322666873534521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,15,0.01798933371901512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,15,0.009973333527644476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,31,0.017797333498795826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,31,0.009328000247478485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,63,0.01786133274435997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,63,0.009354666496316591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,127,0.021205333371957142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,127,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,255,0.015066667149464289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,255,0.012058666596810022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,511,0.015226667126019796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,2047,0.026335999369621277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,511,0.012725333372751871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,1023,0.02298133323589961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,1023,0.015749332805474598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,2047,0.03481066723664602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,4095,0.05160533388455709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,4095,0.03136533250411352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,8191,0.07562133173147838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,8191,0.058378666639328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,1,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,16383,0.11396267016728719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,1,0.007311999797821045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,16383,0.08656000097592671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,3,0.010426666587591171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,3,0.007162666569153468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,31,0.00726400005320708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,7,0.010533332824707031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,7,0.00726400005320708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,15,0.007589333380262057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,15,0.010586666564146677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,31,0.010650667051474253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,63,0.010415999839703241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,63,0.007050666958093643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,127,0.010389333590865135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,127,0.00786666696270307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,255,0.013765333841244379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,255,0.009765333185593287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,511,0.0195573332409064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,511,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,1023,0.02812800059715907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,1023,0.018986667195955913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,2047,0.0440533310174942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,2047,0.023562667270501454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,4095,0.06790400048096974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,4095,0.03806933263937632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,8191,0.08842666943868001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,8191,0.0524479995171229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,1,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,7,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,16383,0.1297760009765625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,1,0.006325333068768184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,16383,0.07576533158620198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,3,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,3,0.006394666930039723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,7,0.00624533308049043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,63,0.0063573333124319715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,127,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,15,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,15,0.0064853330453236895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,31,0.010693332801262537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,31,0.006293333445986112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,63,0.010703999549150467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,127,0.006911999856432279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,255,0.02142400046189626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,255,0.009535999968647957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,511,0.02844800055027008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,511,0.015274666249752045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,1023,0.046181331078211464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,1023,0.022053333620230358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,2047,0.06576533118883769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,2047,0.02848000079393387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,4095,0.0855466624101003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,4095,0.04317333300908407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,8191,0.12754133343696594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,8191,0.06694399813810985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,16383,0.21561066309611002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,16383,0.11340266466140747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,1,0.006634666894872983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,7,0.006768000001708667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,3,0.006464000170429547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,31,0.006613333399097125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,15,0.006773333375652631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,63,0.006624000146985054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,127,0.007360000163316727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,1,0.015285332997639975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,3,0.015791999797026317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,31,0.015285332997639975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,7,0.015365333606799444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,511,0.046575998266537987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,15,0.015610666324694952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,1023,0.06133866806825002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,2047,0.0860586663087209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,127,0.015413332730531693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,2047,0.04011200120051702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,255,0.030293333033720653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,4095,0.12702932953834534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,63,0.01532799998919169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,4095,0.06384000182151794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,8191,0.11091199517250061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,8191,0.2160373330116272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,1,0.008799999952316284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,16383,0.2055199940999349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,16383,0.3908426761627197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,1,0.009098666409651438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,3,0.008869333192706108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,3,0.008949333180983862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,7,0.008837333569924036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,7,0.009088000282645226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,15,0.00897066667675972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,15,0.008949333180983862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,31,0.009770666559537252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,31,0.009125333279371262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,63,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,63,0.00892800030608972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,127,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,127,0.010010666524370512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,255,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,255,0.011909333368142446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,511,0.01553600033124288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,511,0.012469333906968435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,1023,0.01676799977819125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,1023,0.013429333766301474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,2047,0.027295999228954315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,2047,0.015509333461523056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,4095,0.03740799923737844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,4095,0.017680000513792038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,8191,0.02165866643190384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,8191,0.021525333325068157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,16383,0.023925334215164185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,16383,0.026047999660174053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,32767,0.02958400050799052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,32767,0.03316800047953924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,1,0.008746666833758354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,1,0.0069386667261521024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,3,0.008767999708652496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,3,0.00725333330531915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,7,0.008736000085870424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,7,0.007093333328763644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,15,0.008842666943868002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,65535,0.041802664597829185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,15,0.0069759997228781385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,65535,0.03625066578388214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,63,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,31,0.009530666594703993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,127,0.007850666840871176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,31,0.00725333330531915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,63,0.00696000022192796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,127,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,255,0.01138666644692421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,255,0.009125333279371262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,511,0.015541333705186844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,255,0.013866666704416275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,511,0.01977066695690155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,511,0.009578666960199675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,1023,0.016501333564519882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,1023,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,2047,0.02735466758410136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,1023,0.025514667232831318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,2047,0.0116799995303154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,4095,0.01580799991885821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,4095,0.013445333888133367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,8191,0.018976000448067982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,8191,0.015834666788578033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,16383,0.019823999454577763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,16383,0.01865600049495697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,32767,0.02491733431816101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,1,0.008682666967312494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,32767,0.02388266722361247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,1,0.0060159998635451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,3,0.00877333308259646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,3,0.005946666623155276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,7,0.00871999996403853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,31,0.005989333614706993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,65535,0.03160000095764796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,63,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,63,0.006111999973654747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,7,0.006026666611433029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,65535,0.030437332888444264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,15,0.008826666822036108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,15,0.006095999851822853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,31,0.009621333330869675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,127,0.011407999942700068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,127,0.006629333520929019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,255,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,255,0.007647999872763951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,511,0.015487999965747198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,511,0.008218666538596153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,1023,0.016597333053747814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,1023,0.008853333070874214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,2047,0.015040000279744467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,2047,0.009653333574533463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,4095,0.015423999478419622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,4095,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,8191,0.01897066707412402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,8191,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,32767,0.03493333359559377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,16383,0.02107200026512146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,16383,0.015450666348139444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,32767,0.021327999730904896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,1,0.00877333308259646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,3,0.006085333103934924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,1,0.005781333272655805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,3,0.008890666688481966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,65535,0.049509331583976746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,65535,0.02826133370399475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,7,0.008874666566650072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,15,0.009429333110650381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,15,0.005872000008821487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,31,0.010170666500926018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,63,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,31,0.006725333631038666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,63,0.005989333614706993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,127,0.011381333072980246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,127,0.006095999851822853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,255,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,255,0.007647999872763951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,511,0.015439999600251516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,511,0.007994666695594788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,1023,0.013760000467300415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,1023,0.008287999778985977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,2047,0.01655999943614006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,2047,0.00933333362142245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,4095,0.015669333438078564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,4095,0.010608000059922537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,8191,0.025829332570234936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,8191,0.013776000589132309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,16383,0.03070933371782303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,16383,0.016842667013406754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,32767,0.05157333115736643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,32767,0.02430933217207591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,1,0.00891733355820179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,1,0.009039999917149544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,3,0.0086666668454806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,65535,0.07899199922879536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,3,0.009088000282645226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,31,0.009775999933481216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,65535,0.0491893341143926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,7,0.008821333448092142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,7,0.009599999835093817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,15,0.009109333157539368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,15,0.008832000195980072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,31,0.00914666677514712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,63,0.011482667177915573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,63,0.009039999917149544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,127,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,127,0.010570666442314783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,255,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,255,0.012170666207869848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,511,0.01588800052801768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,511,0.01314666618903478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,1023,0.02731200059254964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,1023,0.014479999740918478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,2047,0.03473066786924998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,2047,0.016965333372354507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,7,0.0058133335163195925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,4095,0.018837332725524902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,4095,0.01802666609485944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,8191,0.022858666876951855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,8191,0.021717332303524017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,16383,0.024570666253566742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,16383,0.025914666553338368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,32767,0.030565333863099415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,65535,0.03873066604137421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,32767,0.03321066747109095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,1,0.008896000062425932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,1,0.0069386667261521024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,3,0.008858666444818178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,3,0.007258666679263115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,65535,0.04228266576925913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,31,0.007333333293596904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,7,0.008698666468262672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,63,0.0074346667776505155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,7,0.007114666824539502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,15,0.008986666798591614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,15,0.007333333293596904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,31,0.009397333487868309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,63,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,127,0.011530666301647821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,1023,0.010586666564146677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,127,0.007738666608929634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,255,0.011535999675591787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,255,0.009093333035707474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,511,0.015781333049138386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,511,0.010239999741315842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,1023,0.027317332724730175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,2047,0.01565333331624667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,2047,0.011760000139474869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,4095,0.015941333025693893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,4095,0.013338666409254074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,8191,0.019541333119074505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,8191,0.016127999871969223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,16383,0.021920000513394673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,16383,0.01897066707412402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,32767,0.036570665736993156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,1,0.008821333448092142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,3,0.008698666468262672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,32767,0.026778665681680042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,1,0.006229333579540253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,15,0.008943999807039896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,3,0.006394666930039723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,7,0.008757333581646284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,7,0.00625599982837836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,65535,0.051311999559402466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,65535,0.03472533325354258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,15,0.006448000048597653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,31,0.009568000212311745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,31,0.006335999816656113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,63,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,63,0.006197333335876465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,127,0.011365332951148352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,127,0.006495999793211619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,255,0.011621333658695221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,255,0.008080000057816505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,2047,0.015674666812022526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,511,0.015829333414634068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,511,0.008143999924262365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,1023,0.013386666774749756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,1023,0.00871999996403853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,2047,0.009898666913310686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,4095,0.016805333395799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,4095,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,8191,0.014282666146755219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,8191,0.025733334322770435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,16383,0.03142933299144109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,16383,0.017957333475351334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,32767,0.025616000096003216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,1,0.008725333337982496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,32767,0.05163733164469401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,1,0.005888000130653381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,3,0.008789333204428354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,65535,0.051781331499417625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,3,0.005850666513045629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,7,0.00878399983048439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,65535,0.08002133170763652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,7,0.00613866684337457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,15,0.008965333302815756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,15,0.00590933362642924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,31,0.009573333586255709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,255,0.011285333583752314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,31,0.005935999875267346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,63,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,63,0.005754666402935982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,127,0.011541333049535751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,127,0.0064319999267657595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,255,0.007706666365265846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,511,0.012736000120639801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,511,0.007967999825874964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,1023,0.013274667163689932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,1023,0.008421333506703377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,2047,0.02077866718173027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,2047,0.010431999961535135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,4095,0.02359466751416524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,16383,0.054485330979029335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,4095,0.011882666498422623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,8191,0.034815999368826546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,8191,0.016506666938463848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,16383,0.02773333340883255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,1,0.012410666793584824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,32767,0.08054399987061818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,1,0.009429333110650381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,32767,0.040634666879971824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,7,0.01259200026591619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,3,0.012693333129088083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,3,0.009248000259200731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,7,0.00926399976015091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,15,0.012293333808581034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,15,0.009418666362762451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,65535,0.07163733243942261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,31,0.012885333349307379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,65535,0.13660800457000732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,127,0.010165333126982054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,31,0.01028266673286756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,63,0.012357333054145178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,63,0.00933333362142245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,127,0.012261333564917246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,255,0.015957333147525787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,255,0.012266666938861212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,511,0.01461333284775416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,511,0.023711999257405598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,1023,0.03440000116825104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,1023,0.02658133457104365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,2047,0.05565333366394043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,2047,0.037178667883078255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,4095,0.0864533285299937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,4095,0.06648533542950948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,1,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,1,0.007178666690985362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,8191,0.11195199688275655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,3,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,8191,0.08334400256474812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,3,0.00731733317176501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,31,0.011418666690587997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,7,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,7,0.007573333258430163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,15,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,15,0.007471999774376552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,31,0.0074879998962084455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,63,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,63,0.007381333038210869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,127,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,127,0.008218666538596153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,255,0.02144533395767212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,255,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,511,0.029930666089057922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,511,0.01809599995613098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,1023,0.04799466828505198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,1023,0.027679999669392902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,2047,0.0668213317791621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,2047,0.03440000116825104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,1,0.015957333147525787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,4095,0.08868267138799031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,4095,0.051728000243504844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,8191,0.12979732950528464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,1,0.007114666824539502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,8191,0.07772799829641978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,3,0.01552533358335495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,3,0.007189333438873291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,7,0.015728000551462173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,7,0.006837333242098491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,15,0.015824000040690105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,15,0.007055999711155891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,31,0.015813333292802174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,31,0.006815999746322632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,63,0.01573866605758667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,63,0.006911999856432279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,127,0.01565333331624667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,511,0.047594666481018066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,127,0.0080960001796484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,255,0.030746666093667347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,255,0.015429332852363586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,511,0.021856000026067097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,1023,0.06260266900062561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,1023,0.02810666710138321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,2047,0.08574400345484416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,2047,0.04304533203442892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,4095,0.1283466617266337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,4095,0.06697600086530049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,8191,0.11395200093587239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,1,0.025045332809289295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,1,0.009648000200589498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,3,0.025274666647116344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,8191,0.21988266706466675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,3,0.009861333295702934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,7,0.025072000920772552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,7,0.009653333574533463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,15,0.0249493345618248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,15,0.009839999799927076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,127,0.025072000920772552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,31,0.025216000775496166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,255,0.019823999454577763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,255,0.05042133231957754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,31,0.009829333052039146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,63,0.025263999899228413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,63,0.009717333440979322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,127,0.011322667201360067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,511,0.06459733347098033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,511,0.026672000686327618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,1023,0.08330133557319641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,1023,0.04035199930270513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,2047,0.1311199963092804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,2047,0.0651093324025472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,4095,0.22054400046666464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,4095,0.11156800389289856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,1,0.00891733355820179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,1,0.009050666665037474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,8191,0.3973226547241211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,3,0.008992000172535578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,8191,0.20626133680343628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,3,0.009072000160813332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,7,0.00892800030608972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,7,0.00895999992887179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,15,0.009018666421373686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,15,0.009002666920423508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,127,0.009839999799927076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,31,0.009882666791478792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,31,0.009754666437705358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,511,0.012693333129088083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,63,0.011413333316644033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,63,0.009626666704813639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,127,0.011626667032639185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,2047,0.015850666910409927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,255,0.011519999553759893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,255,0.012256000190973282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,511,0.025807999074459076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,1023,0.03001066545645396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,1023,0.013658666362365087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,2047,0.01926400015751521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,4095,0.019973333925008774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,4095,0.01788266624013583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,8191,0.02443733314673106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,8191,0.021749332547187805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,16383,0.0271573339899381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,16383,0.025994665920734406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,32767,0.04696000119050344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,1,0.00701333334048589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,1,0.00901333304742972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,3,0.007130666946371396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,32767,0.03740799923737844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,3,0.00892800030608972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,7,0.008997333546479544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,15,0.0074506668994824094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,7,0.007178666690985362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,15,0.00915733352303505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,65535,0.0641599992911021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,31,0.009898666913310686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,65535,0.05162666738033295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,31,0.007120000198483467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,63,0.011530666301647821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,63,0.007360000163316727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,511,0.02595199892918269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,511,0.009888000165422758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,127,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,127,0.008154666672150293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,255,0.011626667032639185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,255,0.009663999701539675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,1023,0.013669333110253016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,1023,0.010538666198650995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,2047,0.016693333784739178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,2047,0.011786667009194693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,4095,0.01811733345190684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,4095,0.01349866638580958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,8191,0.027450665831565857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,8191,0.018101333330074947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,16383,0.033488000432650246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,16383,0.021914665897687275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,1,0.008943999807039896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,32767,0.05406933526198069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,1,0.006095999851822853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,32767,0.0322826678554217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,3,0.009088000282645226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,3,0.00620266670982043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,7,0.00901333304742972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,7,0.006133333469430606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,65535,0.08196266492207845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,15,0.010224000240365664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,15,0.006271999950210254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,65535,0.06355733176072438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,31,0.009829333052039146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,31,0.006208000083764394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,63,0.01145600030819575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,63,0.006261333202322324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,127,0.011653333902359009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,127,0.006858666737874349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,255,0.01173866664369901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,255,0.007994666695594788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,511,0.012719999998807907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,511,0.008293333152929941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,1023,0.013232000172138214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,1023,0.009152000149091085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,2047,0.0216799999276797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,2047,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,4095,0.02441066751877467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,4095,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,16383,0.029829333225886028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,8191,0.017130666722853977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,8191,0.03588266670703888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,16383,0.05586666862169901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,1,0.009301333377758661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,1,0.005882666756709416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,32767,0.08362133304278056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,32767,0.043040002385775246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,3,0.008976000050703684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,15,0.00921066664159298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,3,0.0058666666348775225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,7,0.009306666751702627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,7,0.005872000008821487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,15,0.005904000252485275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,31,0.0099093330403169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,31,0.006048000107208888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,65535,0.13731732964515686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,63,0.011445333560307821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,65535,0.07348800202210744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,63,0.007877333089709282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,127,0.011525332927703857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,127,0.006389333556095759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,255,0.012618667135636011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,255,0.00766933336853981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,2047,0.011834666132926941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,511,0.01313599944114685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,511,0.00808533343176047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,1023,0.019146667172511418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,1023,0.009599999835093817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,2047,0.027855999767780304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,4095,0.04051200052102407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,4095,0.018437333405017853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,8191,0.05851200222969055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,8191,0.024400000770886738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,16383,0.08938666184743245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,16383,0.04277333120505015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,32767,0.13134933511416116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,1,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,32767,0.065610667069753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,1,0.009482666850090027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,3,0.013072000195582708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,7,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,3,0.00961599995692571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,7,0.009455999980370203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,15,0.013301332791646322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,15,0.009621333330869675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,65535,0.22527466217676798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,31,0.013034666577974955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,65535,0.11126400033632915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,31,0.009637333452701569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,63,0.012997332960367203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,63,0.009503999724984169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,127,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,511,0.0352960005402565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,127,0.010309333602587381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,255,0.02626666675011317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,255,0.013914667069911957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,511,0.024559999505678814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,1023,0.058890665570894875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,1023,0.04456000030040741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,2047,0.08540800213813782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,1,0.008031999692320824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,2047,0.06530133386452992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,1,0.01611199975013733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,4095,0.11105599999427795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,4095,0.0835093359152476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,15,0.016069332758585613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,3,0.01648533344268799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,3,0.008127999802430471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,7,0.01639466608564059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,7,0.007877333089709282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,15,0.008026666939258575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,31,0.007903999959429106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,31,0.0161013330022494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,63,0.016202667107184727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,63,0.008282666405042013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,127,0.016410666207472484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,127,0.00898133342464765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,255,0.031717332700888314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,1023,0.06398400167624156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,255,0.018250666558742523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,511,0.04869333406289419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,511,0.026543999711672466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,1023,0.0366293340921402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,2047,0.0869706670443217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,2047,0.05178666611512502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,1,0.025722667574882507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,1,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,4095,0.13125333189964294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,4095,0.07667733232180278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,3,0.02589333305756251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,3,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,15,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,7,0.025914666553338368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,31,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,7,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,15,0.025983999172846477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,31,0.02569599946339925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,63,0.02566933383544286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,63,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,127,0.026000000536441803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,255,0.05108266572157542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,127,0.013023999830087027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,255,0.021712000171343487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,511,0.06554666658242543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,511,0.029050665597120922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,1023,0.0848479966322581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,1023,0.043136000633239746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,2047,0.1325546701749166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,2047,0.06751466790835063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,1,0.04419200122356415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,1,0.013722666849692663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,4095,0.2254613240559896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,7,0.0441599984963735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,4095,0.11562666296958923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,3,0.04409066836039225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,3,0.013888000200192133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,31,0.013914667069911957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,7,0.013749333719412485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,63,0.043663998444875084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,15,0.04390933116277059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,15,0.01379199946920077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,31,0.044405331214269005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,63,0.013728000223636627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,511,0.07371733089288075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,127,0.01643199970324834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,127,0.04589866598447164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,1023,0.06125866870085398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,255,0.05418133238951365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,255,0.02329600105683009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,511,0.03735466549793879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,1023,0.11945066849390666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,2047,0.2131999929745992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,1,0.010277333358923594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,2047,0.11061867078145345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,1,0.020256000260512035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,4095,0.20325332880020142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,4095,0.39487465222676593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,3,0.020389333367347717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,7,0.020346666375796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,3,0.0102613332370917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,7,0.010298666854699453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,15,0.020096000283956528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,15,0.010133333504199982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,31,0.020954666038354237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,31,0.01028266673286756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,63,0.020373333245515823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,63,0.010368000095089277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,127,0.020474666108687718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,127,0.011434666812419891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,255,0.03862933317820231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,255,0.02383466561635335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,511,0.06112533311049143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,511,0.04509866734345754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,1,0.012709333250919977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,1023,0.08151466647783916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,1,0.026591998835404713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,1023,0.06592000027497609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,3,0.027280000348885853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,3,0.012901333471139273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,7,0.02664533257484436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,7,0.012682666381200155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,15,0.026634665826956432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,31,0.013082666943470636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,15,0.012730666746695837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,31,0.026975999275843304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,63,0.026447998980681103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,127,0.015285332997639975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,63,0.012815999488035837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,127,0.026661333938439686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,255,0.05340266724427541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,511,0.039120001097520195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,255,0.02718399961789449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,511,0.06771199901898702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,1023,0.0872373382250468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,1,0.04539200166861216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,1023,0.05264000097910563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,1,0.015482666591803232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,3,0.04558933277924856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,3,0.01533866673707962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,7,0.04577066500981649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,7,0.01516266663869222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,15,0.045370668172836304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,15,0.01540800059835116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,63,0.015370666980743408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,31,0.046495998899141945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,31,0.015311999867359797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,63,0.04533333579699198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,127,0.046944002310434975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,127,0.018895999838908512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,255,0.0565280020236969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,255,0.025978667040665943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,511,0.07550933460394542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,511,0.04060266663630804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,1023,0.12180266777674358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,1,0.08085333307584126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,3,0.023818666736284893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,1023,0.06442666550477345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,7,0.081194669008255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,1,0.023541333774725597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,3,0.08134933312733968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,7,0.023562667270501454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,15,0.08080000181992848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,15,0.023503998915354412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,31,0.0811413327852885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,31,0.023685333629449207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,63,0.08107733229796092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,63,0.02345066765944163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,127,0.0831573357184728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,127,0.030640001098314922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,511,0.06566399832566579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,255,0.09584533174832661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,255,0.04266133407751719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,511,0.13184533516565958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,1023,0.2172000010808309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,1023,0.11184533437093098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,1,0.03457599878311157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,1,0.017162666966517765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,3,0.03459733227888743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,3,0.017423999806245167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,7,0.034501334031422935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,15,0.017263999829689663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,7,0.017423999806245167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,15,0.03483733286460241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,31,0.03443199892838796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,31,0.017157333592573803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,63,0.034341332813103996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,63,0.01729600007335345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,127,0.0342399999499321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,127,0.019952000429232914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,255,0.06710400183995564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,255,0.053157334526379905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,511,0.08429867029190063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,511,0.0672106643517812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,1,0.04691733419895172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,1,0.0184906671444575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,3,0.04684799909591675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,3,0.018645333747069042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,7,0.04692266881465912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,7,0.01848000039656957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,15,0.04665066798528036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,15,0.018538666268189747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,31,0.046949331959088646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,31,0.018405333161354065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,63,0.046767999728520714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,63,0.01844266677896182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,127,0.0492799977461497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,127,0.023141334454218548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,255,0.059263999263445534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,255,0.03502399971087774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,511,0.0788213312625885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,1,0.08335466186205547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,511,0.049029335379600525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,1,0.026165333886941273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,3,0.08308266599973042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,3,0.026538667579491932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,7,0.083146666487058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,7,0.02573866645495097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,15,0.08336533109347026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,15,0.026575999955336254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,31,0.08348266283671062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,31,0.026634665826956432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,63,0.08436266581217448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,63,0.026848000784715016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,127,0.08562133709589641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,127,0.03789333254098892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,255,0.09977066516876221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,255,0.048885335524876915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,511,0.1365333298842112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,511,0.07020266850789388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,1,0.15081600348154703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,1,0.04779199759165446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,3,0.048122664292653404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,3,0.15187199910481772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,7,0.15178133050600687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,7,0.048026666045188904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,15,0.04766400158405304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,15,0.15124799807866415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,31,0.15293332934379578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,31,0.04827733337879181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,127,0.15372266372044882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,63,0.1546239952246348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,255,0.0757013310988744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,63,0.047839999198913574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,1,0.0085333331177632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,127,0.05668266614278158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,255,0.17878933747609457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,511,0.24690133333206177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,511,0.11659733454386394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,1,0.00890666681031386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,3,0.008639999975760778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,3,0.009679999823371569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,7,0.008623999853928884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,7,0.009183999771873156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,15,0.009685333197315535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,15,0.008986666798591614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,31,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,31,0.009503999724984169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,63,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,63,0.009519999846816063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,127,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,127,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,255,0.012928000340859095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,255,0.01190399999419848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,511,0.017808000246683758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,511,0.013034666577974955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,4095,0.017978666971127193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,1023,0.015919999529918034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,8191,0.03249066571394602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,1023,0.013818666338920593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,2047,0.01929066702723503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,16383,0.04065066576004028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,2047,0.016634666671355564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,4095,0.02146133283774058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,8191,0.02513599892457326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,16383,0.03102933367093404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,32767,0.06648000081380208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,1,0.008687999720374743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,3,0.007477333148320516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,1,0.007120000198483467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,32767,0.058245331048965454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,3,0.008746666833758354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,7,0.008645333349704742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,7,0.0075519997626543045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,15,0.009392000113924345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,15,0.007071999832987785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,65535,0.09981333216031392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,31,0.007376000285148621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,31,0.011343999455372492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,63,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,65535,0.07923200229803722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,63,0.007344000041484833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,511,0.009706666693091393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,127,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,127,0.007941333577036858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,255,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,255,0.00956266683836778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,511,0.013045333325862885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,1023,0.013349333157142004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,4095,0.015568000574906668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,1023,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,2047,0.021877333521842957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,2047,0.013546666751305262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,4095,0.024901332954565685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,16383,0.03717333326737086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,8191,0.03710933278004328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,8191,0.020853333175182343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,16383,0.05635199944178263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,32767,0.08459200461705525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,1,0.008474666625261307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,1,0.006101333225766818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,3,0.008810666700204214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,32767,0.05351999898751577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,3,0.006384000182151794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,7,0.00879466657837232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,65535,0.13869866728782654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,7,0.006048000107208888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,65535,0.08829866846402486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,15,0.009269333134094873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,15,0.0064266665528217954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,31,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,31,0.006400000303983688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,63,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,63,0.006074666976928711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,127,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,127,0.006842666616042455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,255,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,255,0.008042666440208754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,511,0.01333333303531011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,511,0.008458666503429413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,1023,0.02093333254257838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,1023,0.010277333358923594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,2047,0.028351999819278717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,2047,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,4095,0.04036800066630045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,16383,0.04580266773700714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,4095,0.019946667055288952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,8191,0.05890666445096334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,8191,0.02571733295917511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,16383,0.09008533755938213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,32767,0.13261333107948303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,3,0.010512000570694605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,32767,0.06952000161012013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,1,0.010591999938090643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,1,0.006053333481152852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,3,0.0058666666348775225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,15,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,7,0.006031999985376994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,7,0.010330666477481524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,15,0.006144000217318535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,31,0.010591999938090643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,65535,0.2265066703160604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,31,0.005904000252485275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,63,0.010288000106811523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,65535,0.11322133739789327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,63,0.006085333103934924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,127,0.010298666854699453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,127,0.006586666529377301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,255,0.013530666629473368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,255,0.007887999837597212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,511,0.019354666272799175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,4095,0.06704000135262807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,511,0.008943999807039896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,1023,0.02788266787926356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,1023,0.014058666924635569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,2047,0.042650664846102394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,2047,0.017802666872739792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,4095,0.026672000686327618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,8191,0.08778132994969685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,8191,0.04101333270470301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,16383,0.12760000427563986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,16383,0.0653599997361501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,32767,0.21633066733678183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,32767,0.11339199542999268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,1,0.06305600206057231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,fp8,1,0.03290133426586787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,fp8,3,0.032261334359645844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,65535,0.2079626719156901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,65535,0.3947840134302775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,7,0.06351999938488007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,fp8,7,0.03469333300987879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,3,0.06542400022347768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,15,0.06303999821345012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,fp8,15,0.033520000676314034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,31,0.06348266700903575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,fp8,31,0.03294399877389272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,fp8,63,0.032058666149775185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,63,0.06516266862551372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,127,0.06648000081380208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,fp8,127,0.04234666625658671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,255,0.07831466694672902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,fp8,255,0.0614879975716273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,1,0.08588266372680664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,fp8,1,0.03702933341264725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,3,0.08656533559163411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,fp8,3,0.035829332967599235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,7,0.08691733082135518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,fp8,7,0.036373332142829895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,15,0.08683199683825175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,fp8,15,0.03494933247566223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,31,0.08821333448092143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,fp8,31,0.035760000348091125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,63,0.08834667007128398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,fp8,63,0.035786665976047516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,127,0.08974400162696838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,fp8,127,0.05530133346716563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,255,0.10610666871070862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,fp8,255,0.06473066906134288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,1,0.15585066874821982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,fp8,1,0.057999998331069946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,3,0.1572053333123525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,fp8,3,0.05801600217819214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,fp8,7,0.05775466561317444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,7,0.15611199537913004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,15,0.15920000274976095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,fp8,15,0.05794133245944977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,31,0.1605280041694641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,fp8,31,0.05798399945100149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,fp8,63,0.057146668434143066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,63,0.15964800119400024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,127,0.1602133313814799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,fp8,127,0.06901333232720692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,255,0.18661866585413614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,fp8,255,0.08694400389989217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,fp8,1,0.09045333663622539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,1,0.29524799187978107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,fp8,3,0.09021332859992981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,3,0.2943519949913025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,7,0.29812800884246826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,fp8,7,0.0906880001227061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,fp8,15,0.09058666229248047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,15,0.299727996190389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,fp8,31,0.09035733342170715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,31,0.3028320074081421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,63,0.298634668191274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,fp8,63,0.09055466453234355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,fp8,127,0.10579199592272441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,127,0.2964640061060588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,fp8,255,0.14229333400726318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,255,0.3469333251317342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,1,0.1223360002040863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,fp8,1,0.09160000085830688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,3,0.122789333264033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,fp8,3,0.09151466687520345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,7,0.12289067109425862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,fp8,7,0.09119466940561931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,15,0.12326932946840923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,fp8,15,0.0916373332341512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,31,0.12283200025558472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,fp8,31,0.09186666210492452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,63,0.12218667070070903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,fp8,63,0.09128533800443013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,127,0.12292800347010295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,1,0.16589333613713583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,fp8,127,0.10160000125567119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,fp8,1,0.08344533046086629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,3,0.16609066724777222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,fp8,3,0.08364267150561015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,7,0.1671733260154724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,fp8,7,0.0831573357184728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,fp8,15,0.08361066381136577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,15,0.16843199729919434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,31,0.16734933853149414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,fp8,31,0.08356799681981404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,63,0.1663146714369456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,fp8,63,0.08349866668383281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,127,0.16895999511082968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,fp8,127,0.09954133629798889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,fp8,1,0.10727999607721965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,1,0.31065599123636883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,fp8,3,0.10831999778747559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,3,0.3088746666908264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,7,0.3100159962972005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,fp8,7,0.10834133625030518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,fp8,15,0.1077280044555664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,15,0.31004265944163006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,fp8,31,0.10782399773597717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,31,0.31126399834950763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,63,0.311301330725352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,fp8,63,0.10679999987284343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,fp8,127,0.12878400087356567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,127,0.3115040063858032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,fp8,1,0.17414400974909464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,1,0.5874240001042684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,3,0.590117335319519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,fp8,3,0.17430400848388672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,fp8,7,0.17442667484283447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,7,0.5932266712188721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,fp8,15,0.17427200078964233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,15,0.5955413182576498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,1,0.013514666507641474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,1,0.00921066664159298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,3,0.013957332819700241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,fp8,31,0.17434666554133096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,63,0.5875680049260458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,31,0.5921493371327718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,fp8,127,0.20588799317677817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,7,0.013845333208640417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,7,0.009232000137368837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,15,0.01498666654030482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,15,0.009258666386206945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,fp8,63,0.17412267128626505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,3,0.009695999945203463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,127,0.5796693166097006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,31,0.018186666071414948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,31,0.009343999748428663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,63,0.01850133389234543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,63,0.009103999783595404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,127,0.018698666244745255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,127,0.010570666442314783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,1023,0.01543466622630755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,255,0.021856000026067097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,255,0.011952000359694162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,511,0.015365333606799444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,511,0.012608000387748083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,1023,0.013909333695967993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,2047,0.025306666890780132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,8191,0.047184000412623085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,2047,0.017509333789348602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,4095,0.029909332593282063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,4095,0.02093333254257838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,8191,0.03620799879233042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,16383,0.06756799916426341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,1,0.013781332721312841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,32767,0.10689066847165425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,16383,0.051226665576299034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,1,0.007120000198483467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,3,0.013999999811251959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,3,0.007296000296870868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,32767,0.08423466483751933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,7,0.013872000078360239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,7,0.007407999907930692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,63,0.018426666657129925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,15,0.015130666395028433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,15,0.007151999821265538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,31,0.018346666047970455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,31,0.007194666812817256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,63,0.007637333124876022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,127,0.018986667195955913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,127,0.007680000116427739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,1023,0.019530666371186573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,255,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,255,0.00938666673998038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,511,0.013418667018413544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,4095,0.04189866781234741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,511,0.010255999863147736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,1023,0.012351999680201212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,8191,0.033957332372665405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,2047,0.028389332195123036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,2047,0.015610666324694952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,4095,0.024357333779335022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,8191,0.05982399980227152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,16383,0.09096533060073853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,16383,0.0562666654586792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,1,0.01032533310353756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,1,0.0063786668082078295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,3,0.010581333190202713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,15,0.006325333068768184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,32767,0.1329919993877411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,32767,0.08062933385372162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,3,0.006213333457708359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,7,0.010640000303586325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,7,0.0063786668082078295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,15,0.01028266673286756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,31,0.010533332824707031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,31,0.006309333567818006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,63,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,63,0.006448000048597653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,127,0.010496000448862711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,127,0.00690133310854435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,255,0.013445333888133367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,255,0.00814933329820633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,511,0.01926400015751521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,511,0.009653333574533463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,1023,0.02845333268245061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,1023,0.01573333392540614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,2047,0.04334933559099833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,2047,0.019039999693632126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,4095,0.06783466537793477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,16383,0.12843733032544455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,4095,0.02906133234500885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,8191,0.08860266208648682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,8191,0.04378666480382284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,16383,0.06779733300209045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,1,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,1,0.006165333092212677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,7,0.006064000229040782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,3,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,32767,0.2184106707572937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,3,0.005888000130653381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,32767,0.11481600006421407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,7,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,15,0.01071999967098236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,15,0.00624533308049043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,31,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,31,0.007151999821265538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,63,0.01055466632048289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,63,0.005877333382765452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,127,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,127,0.006458666796485583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,255,0.02075200031201045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,255,0.008938666433095932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,511,0.028223998844623566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,511,0.01360000049074491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,1023,0.04614933331807455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,1023,0.01998399943113327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,2047,0.06462400158246358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,2047,0.025973332424958546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,4095,0.08540266752243042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,4095,0.04012800008058548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,8191,0.12693867087364197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,8191,0.06400533517201741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,16383,0.21342400709788004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,16383,0.11126933495203654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,1,0.013749333719412485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,1,0.013855999956528345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,1,0.0064266665528217954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,1,0.006698666761318843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,32767,0.3907253344853719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,3,0.013850666582584381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,32767,0.204367995262146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,3,0.013914667069911957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,3,0.006810666372378667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,3,0.006400000303983688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,7,0.014186666657527288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,7,0.014175999909639359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,7,0.006805333619316419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,7,0.006677333265542984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,31,0.015477333217859268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,15,0.0141546664138635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,15,0.014170666535695394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,15,0.0064319999267657595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,15,0.0069333333522081375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,63,0.0064213331788778305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,31,0.015210667004187902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,31,0.006858666737874349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,31,0.006858666737874349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,63,0.018394666413466137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,63,0.0183999997874101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,63,0.006858666737874349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,127,0.018474667022625606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,127,0.018650667121013004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,127,0.007413333281874657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,127,0.007397333160042763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,255,0.011509332805871964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,255,0.011674666156371435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,255,0.008314666648705801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,255,0.008645333349704742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,511,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,511,0.011663999408483505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,511,0.008069333309928576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,511,0.00901333304742972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,1023,0.01637866720557213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,1023,0.017152000218629837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,1023,0.0086666668454806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,1023,0.010634666929642359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,2047,0.019434666881958645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,2047,0.024447999894618988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,2047,0.00867733359336853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,2047,0.013167999684810638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,4095,0.024373332659403484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,4095,0.03306666761636734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,4095,0.008618666479984919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,4095,0.017509333789348602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,8191,0.02808533360560735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,8191,0.046021332343419395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,16383,0.00916800027092298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,8191,0.008496000121037165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,8191,0.027866666515668232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,16383,0.03512533257404963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,16383,0.07020799815654755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,16383,0.03985599925120672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,1,0.009818666925032934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,1,0.01009599988659223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,32767,0.03524799893299738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,32767,0.009178666397929192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,32767,0.09318400422732036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,1,0.0058666666348775225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,32767,0.06549333532651265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,7,0.0100853331387043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,1,0.006133333469430606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,3,0.010048000141978264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,3,0.010101333260536194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,3,0.0058080001423756284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,3,0.006064000229040782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,7,0.009861333295702934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,15,0.005957333371043205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,7,0.005935999875267346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,7,0.006506666541099548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,15,0.010090666512648264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,15,0.010069333637754122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,15,0.006144000217318535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,31,0.010101333260536194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,31,0.009775999933481216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,31,0.00578666664659977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,31,0.005914666379491488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,63,0.009808000177145004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,63,0.009914666414260864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,63,0.006079999729990959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,63,0.006117333347598712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,127,0.00980266680320104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,127,0.009914666414260864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,255,0.007749333356817563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,127,0.006234666953484218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,127,0.00655466690659523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,255,0.012074666718641916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,255,0.0120319997270902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,511,0.008517333616813024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,255,0.007365333537260692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,511,0.01632533346613248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,511,0.01687466725707054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,511,0.00749333327015241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,1023,0.021295999487241108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,1023,0.024405332903067272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,1023,0.007834666719039282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,1023,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,2047,0.027589333554108936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,2047,0.03607466568549474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,2047,0.008117333054542542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,2047,0.016821333517630894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,4095,0.034287999073664345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,4095,0.0531626691420873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,4095,0.008416000132759413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,4095,0.021104000508785248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,8191,0.034245334565639496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,8191,0.06805333495140076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,8191,0.008336000144481659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,8191,0.02996266633272171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,16383,0.03443733354409536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,16383,0.047226667404174805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,16383,0.09054932991663615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,16383,0.008512000242869059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,1,0.01022933361430963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,1,0.010175999874869982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,1,0.00624533308049043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,3,0.010165333126982054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,1,0.005984000240763028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,3,0.010421333213647207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,32767,0.03418133407831192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,32767,0.13357866803805032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,32767,0.008496000121037165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,7,0.005914666379491488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,32767,0.07941333452860515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,7,0.006170666466156642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,3,0.006271999950210254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,3,0.00596266674498717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,7,0.009919999788204828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,7,0.010266666611035665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,31,0.010181333248813948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,15,0.01022933361430963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,15,0.010314666976531347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,15,0.006064000229040782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,15,0.0063040001938740415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,63,0.006079999729990959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,31,0.01003200002014637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,31,0.005999999741713206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,31,0.006309333567818006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,63,0.01009599988659223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,63,0.010117333382368088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,255,0.017845333864291508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,63,0.006490666419267654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,127,0.010175999874869982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,127,0.010079999764760336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,127,0.006325333068768184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,127,0.006784000123540561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,255,0.01812800019979477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,255,0.007573333258430163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,255,0.0080960001796484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,511,0.022410665949185688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,511,0.025114665428797405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,511,0.008245333408315977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,511,0.009557333464423815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,1023,0.03246400008598963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,1023,0.03967999915281931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,1023,0.00854399986565113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,1023,0.016117333124081295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,2047,0.03515733281771342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,2047,0.05184000233809153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,2047,0.008613333106040955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,2047,0.019861333072185516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,4095,0.03494933247566223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,4095,0.06692799925804138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,4095,0.008602666358153025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,4095,0.027903998891512554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,8191,0.034976000587145485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,8191,0.0886240005493164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,8191,0.008645333349704742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,8191,0.04520533482233683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,16383,0.03505066782236099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,16383,0.13355200489362082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,16383,0.008623999853928884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,16383,0.07750399907430013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,32767,0.0352906659245491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,32767,0.008682666967312494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,32767,0.22174400091171265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,32767,0.14190399646759033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,1,0.006522666662931442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,3,0.006394666930039723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,3,0.006351999938488007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,1,0.006544000158707301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,7,0.006464000170429547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,7,0.006506666541099548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,1,0.014416000495354334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,1,0.01469333345691363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,3,0.014256000518798828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,31,0.014346666634082794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,31,0.014287999520699183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,3,0.014303999642531076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,7,0.01444799949725469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,7,0.014080000420411428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,63,0.014240000396966934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,63,0.013957332819700241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,63,0.006565333033601443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,63,0.006250666454434395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,127,0.014335999886194864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,127,0.014021333307027817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,127,0.00696000022192796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,127,0.0069759997228781385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,255,0.026330667237440746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,15,0.014368000129858652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,255,0.02622933437426885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,255,0.008234666660428047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,15,0.014442666123310724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,255,0.009402666861812273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,511,0.035818666219711304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,511,0.040048000713189445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,511,0.0099093330403169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,511,0.015594666202863058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,1023,0.035989334185918175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,1023,0.049584001302719116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,1023,0.009989333028594652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,1023,0.01978133370478948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,2047,0.03855466594298681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,2047,0.06710933148860931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,2047,0.009872000043590864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,2047,0.027952000498771667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,4095,0.03851199895143509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,4095,0.08964266379674275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,4095,0.00980266680320104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,4095,0.044997334480285645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,8191,0.03846399982770284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,8191,0.13378666838010153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,8191,0.00973866693675518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,8191,0.07702933251857758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,16383,0.03852266569932302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,16383,0.010090666512648264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,16383,0.22288533051808676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,16383,0.14153066277503967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,15,0.006453333422541618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,1,0.007642666498819987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,32767,0.03868266691764196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,1,0.007711999739209811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,1,0.0063040001938740415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,1,0.00749333327015241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,3,0.0075573331365982694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,32767,0.009952000031868616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,15,0.006741333131988843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,3,0.007706666365265846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,32767,0.2695840001106262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,32767,0.3968586524327596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,3,0.006405333057045937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,3,0.00679466687142849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,7,0.007685333490371704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,7,0.007733333234985669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,7,0.006464000170429547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,15,0.007584000006318092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,7,0.006655999769767125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,31,0.006442666674653689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,15,0.00785600021481514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,15,0.00666133314371109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,31,0.0064106664309899015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,15,0.006522666662931442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,31,0.007903999959429106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,31,0.007807999849319458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,31,0.006618666773041089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,31,0.006501333167155583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,63,0.00884799969693025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,63,0.00854399986565113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,63,0.006533333410819371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,63,0.006538666784763336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,127,0.008725333337982496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,127,0.008656000097592672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,255,0.00871999996403853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,127,0.007216000308593114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,127,0.007296000296870868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,511,0.009648000200589498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,255,0.009077333534757296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,255,0.008698666468262672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,255,0.007914666707317034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,511,0.008703999842206636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,511,0.008031999692320824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,511,0.008997333546479544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,1023,0.013807999591032663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,1023,0.015562667200962702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,1023,0.0081386665503184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,1023,0.009461333354314169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,4095,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,2047,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,2047,0.017488000293572743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,2047,0.007978666573762894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,2047,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,4095,0.016890666137139004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,4095,0.007786666974425316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,4095,0.012058666596810022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,8191,0.012800000607967377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,8191,0.014773332824309668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,8191,0.007749333356817563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,8191,0.01516266663869222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,16383,0.014005333185195923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,16383,0.007733333234985669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,16383,0.01590399940808614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,16383,0.017429333180189133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,32767,0.015034666905800501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,32767,0.018538666268189747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,32767,0.00785600021481514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,32767,0.022431999444961548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,65535,0.01664000004529953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,65535,0.007925333455204964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,65535,0.021744000415007275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,65535,0.02815466622511546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,1,0.007637333124876022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,1,0.007621333623925845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,1,0.005834666391213735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,1,0.005775999898711841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,3,0.0075626665105422335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,3,0.007818666597207388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,3,0.005658666913708051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,3,0.005557333429654439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,7,0.007823999971151352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,7,0.007727999861041705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,7,0.005695999910434087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,131071,0.021216000119845074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,7,0.005850666513045629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,15,0.007663999994595845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,131071,0.02924266705910365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,15,0.007536000261704127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,15,0.005669333040714264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,15,0.005824000264207522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,131071,0.007871999715765318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,31,0.007925333455204964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,31,0.007903999959429106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,31,0.0058559998869895935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,31,0.005648000165820122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,63,0.008703999842206636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,63,0.008863999818762144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,131071,0.03708266715208689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,63,0.005935999875267346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,63,0.00578666664659977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,127,0.00874133345981439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,127,0.008672000219424566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,127,0.006517333288987477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,127,0.00666133314371109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,255,0.00867733359336853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,255,0.00878399983048439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,255,0.007082666580875714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,255,0.007754666730761528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,511,0.008890666688481966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,511,0.00980266680320104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,511,0.007120000198483467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,511,0.007680000116427739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,2047,0.016607999801635742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,1023,0.013728000223636627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,1023,0.015552000453074774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,1023,0.007274666801095009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,1023,0.008250666782259941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,2047,0.013045333325862885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,2047,0.007157333195209503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,2047,0.009093333035707474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,4095,0.012960000584522883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,4095,0.012741333494583765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,4095,0.007157333195209503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,4095,0.010415999839703241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,8191,0.013354666531085968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,8191,0.015066667149464289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,16383,0.01589866727590561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,8191,0.00721066693464915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,8191,0.012229333321253458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,16383,0.013744000345468521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,16383,0.007151999821265538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,16383,0.014250667144854864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,32767,0.01571200042963028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,32767,0.00721066693464915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,32767,0.018922666708628338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,32767,0.017984000345071156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,65535,0.016789333273967106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,65535,0.022624000906944275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,65535,0.007370666911204656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,1,0.007567999884486198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,65535,0.02232533444960912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,1,0.007823999971151352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,1,0.0058453331391016645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,1,0.005653333539764087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,3,0.007807999849319458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,3,0.007802666475375493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,3,0.00590933362642924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,3,0.005744000275929769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,7,0.007850666840871176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,131071,0.021365332106749218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,7,0.007818666597207388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,7,0.0058080001423756284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,131071,0.0395413339138031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,7,0.005935999875267346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,131071,0.032672000428040825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,15,0.0058133335163195925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,131071,0.007061333085099856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,15,0.007770666852593422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,15,0.007743999982873599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,15,0.0058080001423756284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,31,0.007936000203092894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,31,0.008176000167926153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,31,0.0058133335163195925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,31,0.0058613332609335584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,63,0.008656000097592672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,63,0.008645333349704742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,63,0.0058399997651577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,63,0.005935999875267346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,127,0.008933333059151968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,127,0.008922666932145754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,127,0.0061919999619325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,127,0.006186666587988536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,255,0.008805333326260248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,255,0.00897066667675972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,255,0.006768000001708667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,255,0.007520000139872233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,511,0.008901333436369896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,1023,0.007029333462317784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,511,0.009861333295702934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,511,0.007296000296870868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,511,0.007717333113153775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,1023,0.013776000589132309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,1023,0.015573333948850632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,1023,0.008170666793982187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,2047,0.012309333930412928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,2047,0.0124746672809124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,2047,0.007120000198483467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,2047,0.009093333035707474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,4095,0.012229333321253458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,4095,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,4095,0.006965333595871925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,4095,0.010010666524370512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,16383,0.013568000247081121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,8191,0.013536000003417334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,8191,0.015466666469971338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,8191,0.007109333450595538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,8191,0.011994666109482447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,16383,0.016597333053747814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,16383,0.006906666482488315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,16383,0.01421333352724711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,32767,0.017114666601022083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,32767,0.020037333170572918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,32767,0.026650667190551758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,32767,0.007157333195209503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,65535,0.01912533367673556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,65535,0.03195200115442276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,65535,0.007050666958093643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,1,0.007743999982873599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,65535,0.024495999018351238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,1,0.005744000275929769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,1,0.00573333352804184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,1,0.007600000128149986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,3,0.00786666696270307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,3,0.007791999727487564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,3,0.005946666623155276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,7,0.007727999861041705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,3,0.0058026667684316635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,131071,0.025701334079106648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,15,0.00761600024998188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,7,0.007914666707317034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,131071,0.055029332637786865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,131071,0.007322666545708974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,7,0.005904000252485275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,131071,0.03619199991226196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,7,0.0058559998869895935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,15,0.007818666597207388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,31,0.005722666780153911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,15,0.00573333352804184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,15,0.005973333492875099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,31,0.008074666683872541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,31,0.008325333396593729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,31,0.0058666666348775225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,63,0.008634666601816813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,63,0.008778666456540426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,63,0.005712000032265981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,63,0.005637333417932193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,127,0.008938666433095932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,127,0.008778666456540426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,127,0.006335999816656113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,127,0.006106666599710782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,255,0.008693333094318708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,255,0.008805333326260248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,255,0.006954666847983996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,255,0.007424000029762586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,511,0.00895999992887179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,511,0.009786666681369146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,511,0.00725333330531915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,511,0.007893333211541176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,1023,0.01173866664369901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,2047,0.012789333860079447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,1023,0.011941333611806234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,1023,0.0069386667261521024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,4095,0.013376000026861826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,1023,0.008293333152929941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,2047,0.012330666184425354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,2047,0.006981333096822103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,2047,0.00902399979531765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,4095,0.012576000144084295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,4095,0.0069919998447100324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,4095,0.010357333347201347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,8191,0.015568000574906668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,8191,0.019978666057189304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,8191,0.00702400008837382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,8191,0.01313599944114685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,16383,0.01687466725707054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,16383,0.023285334308942158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,16383,0.007146666447321574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,16383,0.015840000162522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,32767,0.02015999952952067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,32767,0.03526400029659271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,32767,0.006911999856432279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,32767,0.02250666668017705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,65535,0.02418133368094762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,65535,0.053685332338015236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,65535,0.007397333160042763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,65535,0.03234133372704188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,1,0.008058666562040647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,1,0.007994666695594788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,1,0.006565333033601443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,1,0.006581333155433337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,131071,0.03180799881617228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,3,0.008042666440208754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,131071,0.08796800176302592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,3,0.007765333478649457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,131071,0.007600000128149986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,3,0.006458666796485583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,3,0.006613333399097125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,131071,0.06128533184528351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,7,0.007962666451931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,7,0.007642666498819987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,7,0.006778666749596596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,7,0.006405333057045937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,15,0.008010666817426682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,15,0.007813333223263422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,15,0.006538666784763336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,15,0.006511999915043513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,31,0.008261333530147871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,31,0.00790933333337307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,31,0.00655466690659523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,31,0.0064266665528217954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,63,0.00891733355820179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,63,0.008693333094318708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,63,0.006751999879876773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,63,0.006831999868154526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,127,0.00895999992887179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,127,0.008832000195980072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,127,0.007407999907930692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,127,0.007413333281874657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,255,0.009189333145817121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,255,0.008976000050703684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,255,0.008197333042820295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,255,0.008623999853928884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,511,0.00891733355820179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,511,0.009749333063761393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,511,0.008117333054542542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,511,0.008997333546479544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,1023,0.014021333307027817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,2047,0.00810666692753633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,1023,0.01569066693385442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,1023,0.008240000034372011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,1023,0.0099093330403169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,2047,0.013210666676362356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,2047,0.016634666671355564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,2047,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,4095,0.013306666165590286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,4095,0.013455999394257864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,4095,0.008021333565314611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,4095,0.01228800043463707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,8191,0.013845333208640417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,16383,0.008005333443482717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,8191,0.01551466683546702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,8191,0.007696000238259633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,8191,0.014565333724021912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,16383,0.014736000448465347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,16383,0.016634666671355564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,16383,0.017685333887736004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,32767,0.016000000139077503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,32767,0.019610666980346043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,32767,0.008053333188096682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,32767,0.022074667116006214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,65535,0.017925333231687546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,65535,0.023050665855407715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,65535,0.00810666692753633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,1,0.007829333345095316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,65535,0.0283146674434344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,1,0.005935999875267346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,1,0.007887999837597212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,1,0.0058613332609335584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,3,0.00797333319981893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,3,0.008165333420038223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,3,0.005935999875267346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,3,0.00589866687854131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,7,0.0080960001796484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,7,0.008016000191370646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,7,0.006074666976928711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,131071,0.022304000953833263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,7,0.0061493335912625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,131071,0.040805332362651825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,15,0.007967999825874964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,131071,0.04033066580692927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,15,0.007957333077987036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,131071,0.00810666692753633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,15,0.005722666780153911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,15,0.006239999706546466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,31,0.008026666939258575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,31,0.008207999790708223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,31,0.0058666666348775225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,31,0.006101333225766818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,63,0.008816000074148178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,63,0.008826666822036108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,63,0.006277333324154218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,63,0.005941333249211311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,127,0.009008000294367472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,127,0.008922666932145754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,127,0.006784000123540561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,127,0.0063680000603199005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,255,0.00891733355820179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,255,0.009248000259200731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,255,0.007055999711155891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,255,0.007690666864315669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,511,0.008986666798591614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,511,0.010170666500926018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,511,0.007216000308593114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,511,0.007983999947706858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,1023,0.014069333672523499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,1023,0.015626666446526844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,1023,0.00721066693464915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,1023,0.008320000022649765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,2047,0.012597333639860153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,2047,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,2047,0.007018666714429855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,4095,0.010288000106811523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,2047,0.00949866697192192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,4095,0.013072000195582708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,4095,0.01349866638580958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,4095,0.007151999821265538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,8191,0.01381333296497663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,8191,0.016063999384641647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,8191,0.007141333073377609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,8191,0.012527999778588613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,16383,0.01431999976436297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,16383,0.01720000058412552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,16383,0.007178666690985362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,16383,0.014368000129858652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,32767,0.01738133281469345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,32767,0.026549334327379864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,32767,0.0074986666440963745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,32767,0.0206133338312308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,65535,0.03267733256022135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,65535,0.01966933285196622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,65535,0.0074346667776505155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,1,0.007813333223263422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,65535,0.025279998779296875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,1,0.008016000191370646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,1,0.005946666623155276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,1,0.005893333504597346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,3,0.007861333588759104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,3,0.008031999692320824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,3,0.00590933362642924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,3,0.0058666666348775225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,7,0.007925333455204964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,7,0.006069333602984746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,7,0.007946666950980822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,7,0.0057920000205437345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,15,0.007903999959429106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,15,0.00784533346692721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,15,0.005882666756709416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,15,0.006021333237489064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,131071,0.05637866755326589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,131071,0.025989333788553875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,31,0.008047999814152718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,31,0.008063999935984612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,31,0.005941333249211311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,131071,0.03769599894682566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,31,0.005994666367769241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,63,0.008736000085870424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,131071,0.007621333623925845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,63,0.00867733359336853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,63,0.006064000229040782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,63,0.006058666855096817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,127,0.009045333291093508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,127,0.00901333304742972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,127,0.006250666454434395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,127,0.00620266670982043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,255,0.009370666618148485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,255,0.008965333302815756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,255,0.007157333195209503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,255,0.0075626665105422335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,511,0.00903466654320558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,1023,0.007162666569153468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,511,0.009824000298976898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,1023,0.008127999802430471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,2047,0.012896000097195307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,511,0.007184000064929326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,511,0.007818666597207388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,1023,0.011663999408483505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,1023,0.011957333733638128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,4095,0.014175999909639359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,2047,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,2047,0.00701333334048589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,2047,0.009098666409651438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,4095,0.013045333325862885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,4095,0.006874666859706243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,4095,0.010239999741315842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,8191,0.016154666741689045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,16383,0.016986666868130367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,8191,0.020453333854675293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,8191,0.0069866664707660675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,8191,0.01321600005030632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,16383,0.023951999843120575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,16383,0.00725333330531915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,16383,0.01618133361140887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,32767,0.020608000457286835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,32767,0.03547733277082443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,32767,0.007007999966541926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,32767,0.021925332645575207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,65535,0.024800000091393787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,65535,0.05435200035572052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,65535,0.007541333635648091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,65535,0.03283733377854029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,1,0.008010666817426682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,1,0.008080000057816505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,1,0.006010666489601135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,1,0.006213333457708359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,3,0.007983999947706858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,3,0.007903999959429106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,3,0.006058666855096817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,3,0.005877333382765452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,7,0.007925333455204964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,7,0.008000000069538752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,7,0.00613866684337457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,131071,0.03138133386770884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,131071,0.0886293351650238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,15,0.008047999814152718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,7,0.006042666733264923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,131071,0.007391999786098798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,15,0.00785600021481514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,15,0.0058133335163195925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,15,0.005893333504597346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,131071,0.06078400214513143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,31,0.008218666538596153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,31,0.008229333286484083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,31,0.00600533311565717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,31,0.006021333237489064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,63,0.008650666723648706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,63,0.00871999996403853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,63,0.006048000107208888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,63,0.006181333214044571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,127,0.008949333180983862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,127,0.008869333192706108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,127,0.006234666953484218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,127,0.006133333469430606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,255,0.008896000062425932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,255,0.00898133342464765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,255,0.007114666824539502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,255,0.00749333327015241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,511,0.011695999652147293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,511,0.011594666788975397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,511,0.007040000210205714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,511,0.007525333513816197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,1023,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,1023,0.011957333733638128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,1023,0.007087999954819679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,1023,0.008383999889095625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,2047,0.015077333897352219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,2047,0.01720000058412552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,2047,0.007141333073377609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,2047,0.009637333452701569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,4095,0.01552533358335495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,4095,0.01930133377512296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,4095,0.00721066693464915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,4095,0.011418666690587997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,8191,0.018592000007629395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,8191,0.026933332284291584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,8191,0.007141333073377609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,8191,0.014874666929244995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,16383,0.021957332889238994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,16383,0.03750933210055033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,32767,0.059418668349583946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,16383,0.007285333548982938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,16383,0.019706666469573975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,32767,0.027552001178264618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,32767,0.0075040000180403394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,32767,0.03480533262093862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,65535,0.09243200222651164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,65535,0.03404266635576884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,65535,0.007781333600481351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,65535,0.05167999863624573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,1,0.009861333295702934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,1,0.00996800015370051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,1,0.006591999903321266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,1,0.006645333642760913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,3,0.010015999898314476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,3,0.010058666889866194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,3,0.006911999856432279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,3,0.007007999966541926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,7,0.010048000141978264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,7,0.010138666878143946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,7,0.007205333560705185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,7,0.006864000111818314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,15,0.010181333248813948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,131071,0.03714133302370707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,15,0.006810666372378667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,131071,0.14379200339317322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,131071,0.007717333113153775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,15,0.010181333248813948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,15,0.006890666360656421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,131071,0.09180266658465068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,31,0.010149333626031876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,31,0.010015999898314476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,31,0.006821333120266597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,31,0.006917333230376244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,63,0.010357333347201347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,63,0.010133333504199982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,63,0.006688000013430913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,63,0.006864000111818314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,127,0.0102186668664217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,127,0.009952000031868616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,127,0.007189333438873291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,127,0.007536000261704127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,255,0.012304000556468964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,255,0.01209066684047381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,255,0.008127999802430471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,255,0.009050666665037474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,1023,0.021594665944576263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,511,0.016501333564519882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,1023,0.024682665864626568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,511,0.017093333105246227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,511,0.008639999975760778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,511,0.009898666913310686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,1023,0.008976000050703684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,1023,0.012810666114091873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,2047,0.027535999814669292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,2047,0.03602666656176249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,2047,0.00926399976015091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,2047,0.020341333001852036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,4095,0.03453866640726725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,4095,0.053504000107447304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,4095,0.009450666606426239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,4095,0.026320000489552815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,8191,0.03417066733042399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,8191,0.06855999926726024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,8191,0.009194666519761086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,8191,0.03677866607904434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,16383,0.034602666894594826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,16383,0.09117333094278972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,16383,0.009610666582981745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,1,0.010512000570694605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,16383,0.057589332262674965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,1,0.01020800011853377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,1,0.006042666733264923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,1,0.005941333249211311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,3,0.010490667074918747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,3,0.010181333248813948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,3,0.006218666831652324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,3,0.006101333225766818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,7,0.010170666500926018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,7,0.010181333248813948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,7,0.006213333457708359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,7,0.007029333462317784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,15,0.010501333822806677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,15,0.01009599988659223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,15,0.006224000205596288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,15,0.006090666477878888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,31,0.010149333626031876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,31,0.010224000240365664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,31,0.006175999840100606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,31,0.0058559998869895935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,63,0.0102186668664217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,63,0.01032533310353756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,63,0.006266666576266289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,63,0.006058666855096817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,255,0.018016000588734944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,127,0.010117333382368088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,127,0.010122666756312052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,127,0.006645333642760913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,127,0.0064853330453236895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,255,0.018063999712467194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,511,0.008282666405042013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,255,0.007887999837597212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,255,0.0085333331177632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,511,0.02269333352645238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,1023,0.016901332885026932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,511,0.02481066683928172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,511,0.00996800015370051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,1023,0.032933334509531655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,1023,0.03931200007597605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,1023,0.00867733359336853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,2047,0.0354720006386439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,2047,0.0516533354918162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,2047,0.008965333302815756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,2047,0.02029866725206375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,4095,0.03515733281771342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,4095,0.06687999765078227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,4095,0.00860799973209699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,4095,0.028917332490285236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,8191,0.04589866598447164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,8191,0.03516799956560135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,8191,0.08966933687527974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,8191,0.008938666433095932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,16383,0.03523733218510946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,1,0.01434133326013883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,1,0.014554666976133982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,16383,0.00901333304742972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,16383,0.13290666540463766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,1,0.006581333155433337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,16383,0.07806933422883351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,3,0.006234666953484218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,1,0.006288000072042148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,3,0.01440000037352244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,3,0.014639999717473984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,3,0.006474666918317477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,7,0.014469332993030548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,7,0.014357333381970724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,7,0.006197333335876465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,7,0.006277333324154218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,15,0.014384000251690546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,15,0.0145066666106383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,15,0.006437333300709724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,15,0.007541333635648091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,31,0.014405333747466406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,31,0.014362666755914688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,31,0.0063786668082078295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,31,0.0063146669417619705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,63,0.014101333916187286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,63,0.014346666634082794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,63,0.006458666796485583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,63,0.006384000182151794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,127,0.014245333770910898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,127,0.014309333016475042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,127,0.0069226666043202085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,127,0.006800000245372455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,255,0.02609066665172577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,255,0.02624533325433731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,511,0.015546667079130808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,255,0.008469333251317343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,255,0.00943999985853831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,511,0.03588266670703888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,511,0.040037333965301514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,511,0.009754666437705358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,1023,0.035562666753927864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,1023,0.05011733373006185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,1023,0.009904000287254652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,2047,0.027232001225153606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,1023,0.01959466685851415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,2047,0.03826666623353958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,4095,0.009930666536092758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,2047,0.06685866912206014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,2047,0.0100426667680343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,8191,0.03825066735347112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,4095,0.03794133414824804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,4095,0.08924800157546997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,4095,0.04462933540344238
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,8191,0.13385599851608276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,8191,0.009888000165422758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,8191,0.07654400169849396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,16383,0.038165333370367684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,1,0.022543999056021374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,1,0.02279466638962428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,1,0.0069973332186539965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,16383,0.009743999689817429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,16383,0.2232053279876709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,1,0.00684799998998642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,3,0.02257599929968516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,16383,0.14058132966359457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,3,0.007045333584149678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,3,0.022639999787012737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,3,0.0069919998447100324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,7,0.022319999833901722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,7,0.02221333235502243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,7,0.007050666958093643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,7,0.00701333334048589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,15,0.02290133386850357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,31,0.0069973332186539965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,15,0.022618666291236877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,15,0.006927999978264173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,15,0.007344000041484833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,31,0.02233600119749705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,31,0.022405333817005157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,31,0.006954666847983996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,63,0.02258666604757309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,63,0.022341333329677582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,63,0.007071999832987785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,63,0.006965333595871925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,127,0.02239999920129776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,127,0.022197333474953968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,127,0.00795199970404307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,127,0.008010666817426682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,255,0.04349866509437561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,255,0.04271999994913737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,255,0.013487999637921652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,255,0.015344000111023584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,511,0.04289066791534424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,511,0.051407997806866966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,511,0.013653332988421122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,511,0.019546666493018467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,1023,0.04308266441027323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,1023,0.06529599924882253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,1023,0.013440000514189402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,1023,0.027503999571005504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,2047,0.045253331462542214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,2047,0.09115733702977498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,2047,0.013690666606028875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,2047,0.04506133496761322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,4095,0.0455626646677653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,4095,0.1353493332862854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,4095,0.013365333278973898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,4095,0.07655466596285503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,8191,0.04568533102671305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,8191,0.013418667018413544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,8191,0.22468799352645874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,8191,0.14028799533843994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,16383,0.046053335070610046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,1,0.007829333345095316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,1,0.008133333176374435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,1,0.006549333532651265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,16383,0.013290667285521826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,1,0.006480000292261441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,3,0.007797333101431529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,16383,0.398357351620992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,3,0.008176000167926153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,16383,0.2678826649983724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,3,0.006746666505932808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,7,0.006506666541099548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,3,0.006506666541099548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,7,0.007637333124876022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,7,0.007829333345095316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,7,0.006906666482488315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,15,0.007818666597207388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,15,0.007786666974425316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,15,0.006415999804933866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,15,0.006586666529377301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,31,0.007813333223263422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,31,0.00808533343176047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,31,0.007034666836261749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,31,0.006704000135262807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,63,0.008687999720374743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,63,0.00867733359336853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,63,0.006645333642760913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,63,0.006405333057045937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,127,0.008805333326260248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,127,0.009109333157539368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,127,0.00726400005320708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,127,0.0074453335255384445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,255,0.008826666822036108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,255,0.009072000160813332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,255,0.008629333227872849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,255,0.008101333553592363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,511,0.008693333094318708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,511,0.010015999898314476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,511,0.008021333565314611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,511,0.009141333401203156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,2047,0.01422400027513504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,1023,0.013946666071812311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,1023,0.01588800052801768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,1023,0.00785600021481514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,1023,0.009888000165422758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,4095,0.008000000069538752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,2047,0.013765333841244379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,2047,0.00821333316465219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,2047,0.011813333878914515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,4095,0.014271999398867289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,4095,0.014538666854302088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,4095,0.012063999970753988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,8191,0.014767999450365702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,8191,0.017130666722853977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,8191,0.00789866658548514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,8191,0.014842666685581207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,16383,0.015263999501864115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,16383,0.018357332795858383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,32767,0.02882133424282074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,16383,0.007861333588759104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,16383,0.01758933315674464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,32767,0.01889066646496455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,32767,0.007946666950980822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,65535,0.020992000897725422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,32767,0.02476799984773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,65535,0.03516799956560135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,65535,0.03130666663249334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,65535,0.00808533343176047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,1,0.008010666817426682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,1,0.007957333077987036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,1,0.0058613332609335584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,1,0.0058559998869895935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,3,0.008026666939258575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,3,0.008053333188096682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,3,0.0057920000205437345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,131071,0.0273333340883255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,131071,0.007967999825874964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,131071,0.0584746648867925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,131071,0.046896000703175865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,3,0.006085333103934924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,7,0.007967999825874964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,7,0.007600000128149986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,7,0.00578666664659977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,7,0.005653333539764087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,15,0.007733333234985669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,15,0.007786666974425316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,15,0.006021333237489064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,15,0.006117333347598712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,31,0.007893333211541176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,31,0.008010666817426682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,31,0.0058026667684316635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,31,0.005775999898711841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,63,0.008586666857202848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,63,0.008954666554927826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,63,0.006085333103934924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,63,0.0058026667684316635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,127,0.008837333569924036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,127,0.008874666566650072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,127,0.006330666442712148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,127,0.006330666442712148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,255,0.009093333035707474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,255,0.009077333534757296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,255,0.00720000018676122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,255,0.007621333623925845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,511,0.008901333436369896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,1023,0.007040000210205714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,511,0.009882666791478792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,511,0.007109333450595538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,511,0.007967999825874964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,1023,0.011658667276302973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,1023,0.01163200040658315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,1023,0.008272000278035799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,2047,0.014053333550691605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,2047,0.014005333185195923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,2047,0.007194666812817256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,2047,0.00914666677514712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,4095,0.014010666559139887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,4095,0.014671999961137772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,4095,0.007002666592597961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,4095,0.010453333457310995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,8191,0.01721599946419398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,8191,0.02147199958562851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,8191,0.007093333328763644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,8191,0.013493333011865616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,16383,0.025194667279720306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,16383,0.01817600056529045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,16383,0.007173333317041397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,16383,0.015957333147525787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,32767,0.022997332115968067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,32767,0.021664001047611237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,32767,0.03740799923737844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,32767,0.007205333560705185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,65535,0.025477332373460133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,65535,0.05693866809209188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,1,0.007983999947706858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,65535,0.00720000018676122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,1,0.008090666805704435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,65535,0.03490666548411051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,1,0.005824000264207522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,1,0.005749333028992017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,3,0.008117333054542542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,3,0.007680000116427739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,3,0.005941333249211311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,3,0.005893333504597346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,7,0.007936000203092894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,7,0.007658666620651881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,7,0.005914666379491488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,7,0.0058559998869895935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,131071,0.03262399882078171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,131071,0.0920799970626831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,131071,0.007514666765928268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,15,0.007967999825874964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,15,0.007760000104705493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,15,0.006026666611433029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,15,0.005882666756709416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,131071,0.061199997862180076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,31,0.008010666817426682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,31,0.008197333042820295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,63,0.0058186668902635574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,31,0.005999999741713206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,31,0.006101333225766818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,63,0.0086666668454806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,63,0.008805333326260248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,63,0.005829333638151486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,127,0.00898133342464765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,127,0.009050666665037474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,127,0.006181333214044571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,127,0.006218666831652324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,255,0.009370666618148485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,255,0.008896000062425932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,255,0.007093333328763644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,255,0.007354666789372762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,1023,0.011871999750534693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,511,0.011445333560307821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,511,0.011498666057984034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,511,0.006927999978264173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,511,0.007536000261704127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,1023,0.011727999895811081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,1023,0.00706666645904382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,1023,0.008512000242869059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,2047,0.015717333803574245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,2047,0.01848000039656957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,2047,0.007157333195209503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,2047,0.00966933307548364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,4095,0.016879999389251072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,4095,0.020768000433842342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,8191,0.014597332725922266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,4095,0.007360000163316727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,4095,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,8191,0.019610666980346043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,8191,0.027973333994547527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,8191,0.007167999943097432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,16383,0.023018665611743927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,16383,0.039120001097520195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,16383,0.007674666742483775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,16383,0.019952000429232914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,32767,0.02824000020821889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,32767,0.06118399898211161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,32767,0.00744000015159448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,32767,0.03387200087308884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,65535,0.034789333740870156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,65535,0.0939573347568512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,65535,0.0075626665105422335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,65535,0.05173333485921224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,1,0.008197333042820295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,1,0.007802666475375493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,1,0.0058186668902635574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,1,0.006106666599710782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,3,0.007840000092983246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,3,0.007957333077987036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,3,0.006122666721542676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,3,0.005925333127379417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,7,0.007983999947706858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,7,0.008192000289758047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,7,0.005888000130653381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,131071,0.007802666475375493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,131071,0.03731200098991394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,7,0.005989333614706993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,131071,0.1472053329149882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,15,0.008101333553592363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,131071,0.09231467048327129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,15,0.007786666974425316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,15,0.006159999718268712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,15,0.00596266674498717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,63,0.008672000219424566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,31,0.0080960001796484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,31,0.007914666707317034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,31,0.005994666367769241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,31,0.006282666698098183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,63,0.008778666456540426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,63,0.006111999973654747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,63,0.005989333614706993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,127,0.008863999818762144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,127,0.008976000050703684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,127,0.006277333324154218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,127,0.00620266670982043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,255,0.011727999895811081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,255,0.011418666690587997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,255,0.00721066693464915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,255,0.0075626665105422335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,511,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,511,0.011600000162919363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,1023,0.008858666444818178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,511,0.007093333328763644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,511,0.007696000238259633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,1023,0.016309333344300587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,1023,0.01685333376129468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,1023,0.007424000029762586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,2047,0.018816000471512478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,2047,0.02363733450571696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,2047,0.007658666620651881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,2047,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,4095,0.02351466566324234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,4095,0.032325332363446556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,4095,0.007226666435599327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,4095,0.01358933374285698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,8191,0.02739733209212621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,8191,0.04446933170159658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,8191,0.007770666852593422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,8191,0.022261333962281544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,16383,0.033514666060606636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,16383,0.00772266648709774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,32767,0.008053333188096682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,16383,0.06798933446407318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,16383,0.03071466585000356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,32767,0.034629332522551216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,32767,0.09155199925104777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,32767,0.050954664746920265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,65535,0.008069333309928576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,65535,0.03458133339881897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,65535,0.13868799805641174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,65535,0.08373333017031352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,1,0.010527999450763067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,1,0.010304000228643417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,1,0.006629333520929019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,1,0.006864000111818314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,3,0.010330666477481524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,131071,0.03640000025431315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,3,0.006874666859706243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,3,0.010346666599313417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,131071,0.2392959992090861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,131071,0.008383999889095625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,3,0.006784000123540561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,7,0.010586666564146677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,7,0.01062400018175443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,7,0.0068853336075941724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,7,0.00696000022192796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,131071,0.14473066727320352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,15,0.010426666587591171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,15,0.010506667196750641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,15,0.0068693334857622785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,15,0.006762666627764702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,63,0.010250666489203772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,31,0.01022933361430963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,31,0.010586666564146677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,31,0.006757333253820737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,31,0.006810666372378667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,63,0.010245333115259806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,63,0.006725333631038666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,63,0.0068853336075941724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,127,0.0074506668994824094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,127,0.010122666756312052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,127,0.010474666953086853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,127,0.007296000296870868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,255,0.018170667191346485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,255,0.018085333208243053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,255,0.00854399986565113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,255,0.009599999835093817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,511,0.022709332406520844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,511,0.025055999557177227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,1023,0.00972800018886725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,511,0.009450666606426239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,511,0.011616000284751257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,1023,0.033285332222779594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,1023,0.03978666663169861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,1023,0.020197333147128422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,2047,0.03557866563399633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,2047,0.05283733208974203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,2047,0.00956266683836778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,2047,0.024362665911515553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,4095,0.035674666364987694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,4095,0.009583999713261923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,4095,0.06796800096829732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,4095,0.03405333310365677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,8191,0.03596800069014231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,1,0.014544000228246054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,8191,0.09071999788284302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,1,0.014602666099866232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,8191,0.009797333429257074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,1,0.006218666831652324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,8191,0.055567999680837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,1,0.006298666819930077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,3,0.014501333236694336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,3,0.014677333335081736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,7,0.00625599982837836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,3,0.006277333324154218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,3,0.006517333288987477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,7,0.014352000008026758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,7,0.014576000471909841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,7,0.006362666686375936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,15,0.01461333284775416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,15,0.014720000326633453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,15,0.006234666953484218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,15,0.006224000205596288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,31,0.014896000425020853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,31,0.014325333138306936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,31,0.00625599982837836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,31,0.0064266665528217954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,63,0.014378666877746582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,63,0.014645333091417948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,63,0.006261333202322324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,255,0.02677333354949951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,63,0.006229333579540253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,127,0.014639999717473984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,127,0.014186666657527288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,127,0.006981333096822103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,127,0.007007999966541926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,255,0.026714667677879333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,255,0.0084906667470932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,255,0.009786666681369146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,511,0.036144000788529716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,511,0.040421334405740104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,511,0.009952000031868616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,511,0.01599466676513354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,1023,0.03595733394225439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,1023,0.04977599779764811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,1023,0.010079999764760336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,1023,0.02015999952952067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,2047,0.0383093332250913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,2047,0.06830933193365733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,2047,0.010010666524370512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,2047,0.027989332874615986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,4095,0.03850133220354716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,4095,0.09108799695968628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,4095,0.00997866690158844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,4095,0.04621333380540212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,8191,0.038762666285037994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,8191,0.13493333260218301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,8191,0.010010666524370512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,1,0.022944000860055287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,8191,0.07772266864776611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,1,0.02292266736427943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,1,0.00702400008837382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,1,0.007690666864315669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,3,0.023152001202106476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,3,0.022517333428064983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,7,0.006981333096822103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,3,0.007151999821265538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,3,0.007018666714429855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,7,0.022917332748572033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,7,0.022624000906944275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,7,0.007349333415428798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,15,0.02252800017595291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,15,0.0234400009115537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,15,0.007135999699433644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,15,0.007178666690985362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,31,0.02290133386850357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,31,0.02258666604757309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,31,0.007397333160042763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,31,0.006895999734600385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,63,0.022463999688625336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,63,0.022624000906944275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,63,0.007029333462317784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,63,0.007087999954819679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,127,0.022416000564893086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,127,0.02298133323589961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,127,0.007967999825874964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,127,0.008031999692320824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,255,0.04275733232498169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,255,0.04289066791534424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,255,0.013594667116800943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,255,0.015301333119471868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,511,0.0439626673857371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,511,0.05184000233809153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,511,0.01328533391157786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,511,0.019343999524911244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,1023,0.04293866455554962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,1023,0.06568000217278798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,1023,0.013616000612576803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,1023,0.027834666272004444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,2047,0.04597333570321401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,4095,0.136053333679835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,2047,0.09132799506187439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,2047,0.01339200014869372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,2047,0.04474666714668274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,4095,0.045738667249679565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,4095,0.013594667116800943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,4095,0.07699200014273326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,8191,0.04599999884764353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,8191,0.013445333888133367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,8191,0.22601600488026938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,1,0.03875199953715006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,8191,0.14044800400733948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,1,0.038975998759269714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,1,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,1,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,3,0.039274667700131737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,3,0.038704000413417816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,3,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,3,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,7,0.03863999992609024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,7,0.0390133336186409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,7,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,7,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,15,0.03939733405907949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,15,0.038848000268141426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,15,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,15,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,31,0.03858133405447006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,31,0.03893866638342539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,31,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,31,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,63,0.03887466589609782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,63,0.03845333307981491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,63,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,63,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,255,0.040261333187421165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,127,0.03834133346875509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,511,0.040752001106739044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,127,0.03868799904982249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,127,0.012719999998807907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,127,0.012645332763592402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,511,0.012645332763592402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,255,0.04025600105524063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,255,0.012826666235923767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,255,0.01646399994691213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,511,0.05318933228651682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,511,0.025263999899228413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,1023,0.041189332803090416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,1023,0.07679999868075053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,1023,0.012757333616415659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,1023,0.04213866591453552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,2047,0.04418133199214935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,2047,0.1237440009911855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,2047,0.012768000364303589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,2047,0.07461866736412048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,4095,0.044165333112080894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,4095,0.21213332811991373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,4095,0.012837332983811697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,4095,0.1377066671848297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,8191,0.04369066655635834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,1,0.007098666702707608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,8191,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,8191,0.38600532213846844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,1,0.014912000546852747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,8191,0.26714666684468585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,1,0.015178666760524115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,1,0.0069440001000960665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,3,0.014954666296641031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,3,0.014997333288192749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,3,0.007135999699433644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,3,0.007055999711155891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,7,0.01471466695268949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,7,0.007386666412154834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,7,0.014789332946141561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,7,0.007040000210205714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,15,0.014965333044528961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,15,0.014922666052977243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,31,0.014511999984582266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,31,0.00706666645904382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,15,0.007349333415428798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,15,0.00721066693464915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,31,0.015360000232855478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,31,0.007178666690985362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,63,0.014698666830857595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,63,0.01463466634353002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,63,0.006965333595871925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,63,0.007071999832987785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,127,0.01462399959564209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,127,0.014629332969586054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,127,0.00814933329820633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,127,0.007823999971151352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,255,0.026863999664783478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,255,0.027327999472618103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,255,0.009429333110650381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,255,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,511,0.036490666369597115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,511,0.040693332751592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,511,0.011418666690587997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,511,0.019061333189407986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,1023,0.03646933287382126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,1023,0.051269332567850746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,2047,0.011498666057984034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,1023,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,1023,0.024682665864626568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,2047,0.03899200012286504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,2047,0.06870933373769124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,1,0.023391999304294586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,2047,0.03489066660404205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,1,0.023226665953795116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,1,0.007322666545708974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,1,0.006895999734600385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,3,0.023178666830062866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,3,0.023434666295846302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,3,0.006965333595871925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,3,0.007061333085099856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,7,0.023215999205907185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,7,0.023258666197458904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,7,0.007354666789372762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,7,0.00690133310854435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,15,0.02311466634273529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,15,0.02314666658639908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,15,0.0069866664707660675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,15,0.007082666580875714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,31,0.02333866556485494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,31,0.02319466571013133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,31,0.007344000041484833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,31,0.0069333333522081375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,63,0.022890667120615642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,63,0.022986667851607006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,63,0.006965333595871925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,63,0.007151999821265538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,127,0.0230880007147789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,127,0.02293866624434789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,127,0.008517333616813024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,127,0.008090666805704435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,255,0.04322666426499685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,255,0.04334933559099833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,255,0.013663999736309052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,255,0.01586666703224182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,511,0.043653334180514015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,511,0.05322133501370748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,511,0.013983999689420065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,511,0.01987733319401741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,1023,0.04343999922275543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,1023,0.06638399759928386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,1023,0.013674666484196981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,1023,0.029093332588672638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,2047,0.04613866905371348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,2047,0.09303466478983562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,2047,0.013946666071812311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,1,0.039477333426475525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,2047,0.045893331368764244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,1,0.03938133269548416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,1,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,1,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,3,0.03951466580231985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,3,0.04038933416207632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,3,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,3,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,7,0.03926933308442434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,7,0.039333333571751915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,7,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,7,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,15,0.03962666789690653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,15,0.04012266546487808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,15,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,15,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,31,0.03930133332808813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,31,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,63,0.039605334401130676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,31,0.039749334255854286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,31,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,63,0.038880000511805214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,63,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,63,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,127,0.03935466706752777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,127,0.03956266740957896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,127,0.012831999609867731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,127,0.012789333860079447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,255,0.04082666585842768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,255,0.04127999891837438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,255,0.012576000144084295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,255,0.01687466725707054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,511,0.041450666884581246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,511,0.05393599967161814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,511,0.012741333494583765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,511,0.02526933451493581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,1023,0.04105599969625473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,1023,0.07714666426181793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,1023,0.012693333129088083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,1,0.07080533107121785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,1023,0.04270400106906891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,2047,0.04428266485532125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,2047,0.12520000338554382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,2047,0.012800000607967377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,2047,0.07390399773915608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,1,0.07111466427644093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,1,0.01844266677896182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,1,0.018250666558742523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,3,0.07140799860159557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,3,0.07116800049940745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,3,0.01838933303952217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,3,0.018415999909241993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,7,0.07098133365313213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,7,0.07077866792678833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,7,0.018351999421914417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,7,0.01821333294113477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,31,0.07097599903742473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,15,0.07140799860159557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,15,0.07123200098673503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,15,0.01836266616980235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,15,0.018581333259741466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,31,0.0710506687561671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,31,0.018325333793958027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,31,0.018309333672126133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,63,0.0701279987891515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,63,0.07066666583220164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,63,0.018661333868900936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,63,0.018330667167901993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,127,0.07063999772071838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,127,0.07035199801127116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,127,0.02147199958562851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,127,0.021354667842388153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,255,0.07653866708278656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,255,0.07654400169849396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,255,0.02178666740655899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,255,0.029978667696317036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,511,0.07725866635640462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,511,0.09596799810727437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,511,0.02149333308140437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,511,0.046709333856900535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,1023,0.07699733475844066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,1023,0.14160533746083578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,1023,0.021989333132902782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,1023,0.07796800136566162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,2047,0.07905066510041554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,float16,1,0.02383466561635335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,2047,0.02146666745344798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,2047,0.23072532812754312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,2047,0.14004799723625183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,1,0.023904000719388325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,fp8,1,0.00808533343176047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,1,0.007957333077987036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,float16,3,0.023946667710940044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,3,0.023669332265853882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,fp8,3,0.00790933333337307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,3,0.007786666974425316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,float16,7,0.023797333240509033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,7,0.008047999814152718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,7,0.023930666347344715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,fp8,7,0.00808533343176047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,float16,15,0.02402666707833608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,15,0.023589332898457844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,fp8,15,0.00789866658548514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,15,0.007770666852593422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,float16,31,0.023770667612552643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,31,0.02404266595840454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,fp8,31,0.007983999947706858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,31,0.008031999692320824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,float16,63,0.023061332603295643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,63,0.023541333774725597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,fp8,127,0.00943999985853831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,fp8,63,0.007733333234985669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,63,0.00785600021481514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,float16,127,0.023552000522613525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,127,0.023498666783173878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,127,0.009322666873534521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,float16,255,0.04417600234349569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,255,0.044256001710891724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,fp8,255,0.015647999942302704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,255,0.01850133389234543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,float16,511,0.044309332966804504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,511,0.05385600030422211
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,fp8,511,0.015615999698638916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,511,0.02370133250951767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,float16,1023,0.044250667095184326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,1023,0.06753066678841908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,fp8,1023,0.015770666301250458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,1023,0.03501333296298981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,float16,1,0.04036800066630045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,1,0.04060266663630804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,fp8,3,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,fp8,1,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,1,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,float16,3,0.041008000572522484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,3,0.04013866682847341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,3,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,float16,7,0.040362666050593056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,7,0.04137066751718521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,fp8,7,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,7,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,float16,15,0.04015466570854187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,15,0.04015466570854187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,fp8,15,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,15,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,float16,31,0.040192000567913055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,31,0.04063999901215235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,fp8,31,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,31,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,float16,63,0.039621333281199135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,63,0.039674667020638786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,fp8,63,0.011285333583752314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,63,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,float16,127,0.039893334110577904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,127,0.0402453343073527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,fp8,127,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,127,0.013002666334311167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,float16,255,0.04250133534272512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,255,0.04177600145339966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,fp8,255,0.013013333082199097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,255,0.01722666621208191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,float16,511,0.04253333310286204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,511,0.05561066667238871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,fp8,511,0.012885333349307379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,511,0.025962665677070618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,float16,1023,0.04227200150489807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,1023,0.07904000083605449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,fp8,1023,0.013050666699806849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,float16,1,0.07266133526961009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,1023,0.04324266811211904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,1,0.07253333429495494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,fp8,1,0.018272000054518383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,1,0.018298666924238205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,float16,7,0.07239466905593872
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,float16,3,0.07182933390140533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,3,0.07228266696135204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,fp8,3,0.018405333161354065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,3,0.018602666755517323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,7,0.07255466779073079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,fp8,7,0.01833600054184596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,7,0.018357332795858383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,float16,15,0.07231999933719635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,15,0.07187733550866444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,fp8,15,0.018618666877349217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,15,0.01844266677896182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,float16,31,0.07266666491826375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,31,0.07253866891066234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,fp8,31,0.01829333355029424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,31,0.018383999665578205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,float16,63,0.07144000132878621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,127,0.0724480003118515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,63,0.07115200161933899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,127,0.021327999730904896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,fp8,63,0.018522666146357853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,63,0.01844266677896182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,float16,127,0.07223466535409291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,fp8,127,0.021456000705560047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,float16,255,0.07727466523647308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,255,0.07743466893831889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,fp8,255,0.02143466720978419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,255,0.03014933317899704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,float16,511,0.07753066718578339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,511,0.09707732995351155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,fp8,511,0.02186666677395503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,511,0.046911999583244324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,float16,1023,0.07738666733105977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,1023,0.14282666643460593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,fp8,1023,0.021375998854637146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,1023,0.0787306676308314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,float16,1,0.13222933808962503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,1,0.13321600357691446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,fp8,1,0.03214933226505915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,1,0.03224000086386999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,float16,3,0.13306132952372232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,3,0.1321386694908142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,fp8,3,0.03252800057331721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,3,0.03219733387231827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,float16,7,0.1330826679865519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,7,0.13217066725095114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,fp8,7,0.032298666735490165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,7,0.032560000816980995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,float16,15,0.1322719951470693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,15,0.13211199641227722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,fp8,15,0.03264000018437704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,15,0.03215466688076655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,float16,31,0.13331733147303262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,31,0.1320799986521403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,fp8,31,0.03230933348337809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,31,0.032560000816980995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,float16,63,0.13264000415802002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,63,0.13205867012341818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,127,0.040607998768488564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,fp8,63,0.03230933348337809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,63,0.032069332897663116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,float16,127,0.13403200109799704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,127,0.13344533244768778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,fp8,127,0.04014399896065394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,float16,255,0.142384002606074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,255,0.14206399520238241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,fp8,255,0.04070399949947993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,255,0.057087997595469155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,float16,511,0.14282666643460593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,511,0.1814240018526713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,fp8,511,0.040474665661652885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,fp8,1023,0.04138133426507314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,511,0.08662933111190796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,float16,1023,0.14379733800888062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,1023,0.2681493361790975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,1,0.007733333234985669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,3,0.007887999837597212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,1023,0.1485973298549652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,1,0.007823999971151352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,1,0.006415999804933866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,1,0.006805333619316419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,3,0.008005333443482717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,3,0.0064266665528217954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,3,0.006533333410819371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,15,0.007920000081261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,7,0.00784533346692721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,7,0.008053333188096682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,7,0.006528000036875407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,7,0.006448000048597653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,15,0.008165333420038223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,15,0.006437333300709724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,63,0.008746666833758354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,15,0.006458666796485583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,31,0.008122666428486506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,31,0.008234666660428047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,31,0.00679466687142849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,31,0.006490666419267654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,63,0.009109333157539368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,63,0.006453333422541618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,63,0.00666133314371109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,127,0.008863999818762144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,127,0.009018666421373686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,127,0.007391999786098798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,127,0.0074506668994824094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,255,0.008943999807039896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,255,0.009258666386206945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,255,0.007781333600481351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,255,0.008581333483258883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,511,0.008853333070874214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,511,0.01007466639081637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,511,0.008234666660428047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,511,0.009194666519761086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,1023,0.01191466674208641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,1023,0.011839999506870905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,1023,0.007829333345095316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,1023,0.009632000078757605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,2047,0.013712000101804733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,2047,0.014352000008026758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,2047,0.008074666683872541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,2047,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,4095,0.0141546664138635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,4095,0.014970666418472925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,4095,0.007882666463653246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,4095,0.01219733307758967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,8191,0.01693333312869072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,8191,0.021744000415007275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,8191,0.008122666428486506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,8191,0.01653333380818367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,16383,0.018432000031073887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,16383,0.025589334468046825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,16383,0.008143999924262365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,16383,0.01995733380317688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,32767,0.022122666239738464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,32767,0.03707200040419897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,32767,0.008256000156203905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,32767,0.029285334050655365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,65535,0.025818665822347004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,65535,0.057562669118245445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,65535,0.00842666688064734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,65535,0.04215999941031138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,1,0.007957333077987036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,1,0.00790933333337307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,1,0.005759999776879947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,1,0.0058026667684316635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,3,0.00786666696270307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,131071,0.03331200033426285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,3,0.007770666852593422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,3,0.005968000118931134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,131071,0.09242666761080424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,3,0.005930666501323382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,131071,0.00850133349498113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,131071,0.080485333998998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,7,0.007765333478649457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,7,0.007802666475375493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,7,0.005941333249211311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,7,0.005888000130653381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,15,0.00785600021481514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,15,0.008063999935984612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,15,0.0060159998635451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,15,0.0061493335912625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,31,0.008122666428486506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,31,0.008133333176374435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,31,0.00596266674498717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,31,0.00596266674498717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,63,0.008805333326260248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,63,0.008949333180983862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,63,0.005999999741713206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,63,0.0061493335912625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,127,0.008943999807039896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,127,0.00890666681031386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,127,0.006330666442712148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,127,0.006394666930039723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,255,0.008890666688481966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,255,0.009141333401203156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,255,0.00726400005320708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,1023,0.01190399999419848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,255,0.007802666475375493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,1023,0.007194666812817256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,511,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,511,0.011493333925803503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,511,0.007061333085099856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,511,0.007834666719039282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,4095,0.01692266638080279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,1023,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,1023,0.008442666381597519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,2047,0.01639466608564059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,2047,0.018533332894245785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,2047,0.00720000018676122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,2047,0.009808000177145004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,4095,0.020432000358899433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,4095,0.007386666412154834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,4095,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,8191,0.019989332805077236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,8191,0.0074346667776505155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,8191,0.028677334388097126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,32767,0.028533334533373516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,8191,0.015354666858911514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,16383,0.023258666197458904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,16383,0.03932266682386398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,16383,0.007386666412154834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,16383,0.021183999876181286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,32767,0.06192533175150553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,32767,0.00785600021481514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,32767,0.03594133257865906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,65535,0.035002666215101876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,65535,0.09435733159383138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,65535,0.007823999971151352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,65535,0.052426666021347046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,1,0.005797333394487699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,1,0.007813333223263422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,1,0.007962666451931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,1,0.005968000118931134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,3,0.007957333077987036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,3,0.007893333211541176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,131071,0.037402667105197906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,7,0.007925333455204964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,3,0.005984000240763028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,3,0.005882666756709416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,131071,0.14833066860834757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,15,0.007877333089709282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,7,0.007941333577036858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,7,0.00596266674498717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,31,0.008170666793982187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,7,0.0058026667684316635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,131071,0.008133333176374435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,15,0.007930666829148928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,131071,0.09293333689371745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,15,0.00596266674498717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,63,0.00914666677514712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,15,0.0058026667684316635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,31,0.008127999802430471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,31,0.00596266674498717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,31,0.005941333249211311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,63,0.008896000062425932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,63,0.005994666367769241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,63,0.006895999734600385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,127,0.009002666920423508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,127,0.008933333059151968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,127,0.006133333469430606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,127,0.006079999729990959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,255,0.01145600030819575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,255,0.01138666644692421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,511,0.007034666836261749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,255,0.007322666545708974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,1023,0.016042667130629223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,255,0.007157333195209503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,511,0.011525332927703857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,1023,0.00878399983048439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,511,0.011866666376590729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,511,0.0075626665105422335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,1023,0.016821333517630894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,1023,0.0074346667776505155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,2047,0.018789333601792652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,2047,0.023872000475724537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,2047,0.007578666632374127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,2047,0.010703999549150467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,4095,0.0229120006163915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,4095,0.03196800003449122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,4095,0.007711999739209811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,4095,0.013455999394257864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,16383,0.06786133348941803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,8191,0.027552001178264618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,8191,0.007706666365265846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,8191,0.044682666659355164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,8191,0.02183466653029124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,16383,0.03301866600910822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,16383,0.008192000289758047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,32767,0.007776000226537387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,16383,0.03033066789309184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,32767,0.03462400039037069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,32767,0.0920960009098053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,32767,0.051125332713127136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,65535,0.034634667138258614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,65535,0.1386133333047231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,65535,0.008240000034372011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,1,0.009999999776482582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,65535,0.08193066716194153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,1,0.009818666925032934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,1,0.006330666442712148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,1,0.005946666623155276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,3,0.00980266680320104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,131071,0.035936000446478523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,3,0.00966933307548364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,131071,0.23957866430282593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,131071,0.007957333077987036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,3,0.006437333300709724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,3,0.00595199999709924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,131071,0.14574399590492249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,7,0.010015999898314476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,7,0.00972800018886725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,7,0.005888000130653381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,7,0.005925333127379417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,15,0.010122666756312052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,15,0.010255999863147736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,15,0.005968000118931134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,15,0.006079999729990959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,31,0.010053333515922228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,63,0.005824000264207522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,31,0.009743999689817429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,127,0.010037333394090334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,127,0.009765333185593287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,31,0.005935999875267346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,255,0.011765333513418833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,31,0.005935999875267346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,63,0.009941333283980688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,63,0.010138666878143946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,63,0.006144000217318535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,127,0.006165333092212677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,127,0.006458666796485583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,255,0.011749333391586939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,255,0.006949333474040031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,1023,0.0244159996509552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,255,0.00749333327015241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,511,0.01669866715868314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,1023,0.010149333626031876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,511,0.017103999853134155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,2047,0.03572800010442734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,511,0.0075040000180403394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,511,0.008218666538596153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,1023,0.021754667162895203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,1023,0.007477333148320516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,2047,0.02698666602373123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,2047,0.008117333054542542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,2047,0.01642666632930438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,4095,0.034074666599432625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,4095,0.05258666475613912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,4095,0.008234666660428047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,8191,0.02880000074704488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,4095,0.020469332734743755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,8191,0.03387733300526937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,8191,0.0670666644970576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,8191,0.00816000004609426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,16383,0.033861334125200905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,16383,0.08973333239555359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,16383,0.008293333152929941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,16383,0.04611733555793762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,32767,0.033717334270477295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,32767,0.13314132889111838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,32767,0.008176000167926153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,32767,0.07874666651089986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,65535,0.034261333445707955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,65535,0.22211732467015585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,65535,0.008394666636983553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,65535,0.14199466506640115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,131071,0.035349334279696144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,float16,1,0.041434665520985924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,1,0.041706666350364685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,131071,0.008005333443482717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,fp8,1,0.012448000411192576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,131071,0.3999679883321126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,1,0.012351999680201212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,131071,0.2711840073267619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,float16,3,0.04131199916203817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,3,0.041493333876132965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,fp8,3,0.012554666648308435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,fp8,7,0.01240533341964086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,3,0.012549333274364471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,7,0.04127999891837438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,float16,7,0.04173333446184794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,7,0.012624000509579977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,float16,15,0.04124800115823746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,15,0.041450666884581246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,fp8,15,0.012543999900420507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,15,0.012373333175977072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,float16,31,0.041162667175134025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,31,0.04144000013669332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,fp8,31,0.012639999389648438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,31,0.012618667135636011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,float16,63,0.04058133314053217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,63,0.040565334260463715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,fp8,63,0.01240533341964086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,63,0.012282667060693106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,float16,127,0.040565334260463715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,127,0.04055466751257578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,fp8,127,0.015392000476519266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,127,0.015402667224407196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,float16,255,0.04483733574549357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,255,0.04382933179537455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,fp8,255,0.015237333873907724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,255,0.02021866664290428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,511,0.03221333275238673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,float16,511,0.04411733150482178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,1,0.07320533196131389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,511,0.05706666906674703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,fp8,511,0.015386667102575302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,float16,1,0.07309866448243459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,fp8,1,0.018853332847356796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,1,0.018677332748969395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,float16,3,0.0734986662864685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,3,0.0730453332265218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,fp8,3,0.01863466699918111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,3,0.018730666488409042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,7,0.018613333503405254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,15,0.07320533196131389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,float16,7,0.07347199817498525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,fp8,15,0.01865600049495697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,7,0.07382399837176006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,fp8,7,0.018842666099468868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,float16,15,0.07343466579914093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,15,0.018559999763965607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,float16,31,0.07296533385912578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,31,0.07307200133800507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,fp8,31,0.018922666708628338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,31,0.018746666610240936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,float16,63,0.07209066549936931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,63,0.07252799967924754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,fp8,63,0.018698666244745255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,63,0.018565333137909572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,float16,127,0.07372800012429555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,127,0.07323200007279713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,fp8,127,0.022490667800108593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,127,0.02229333420594533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,float16,255,0.07946133116881053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,255,0.07939200103282928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,511,0.10001599788665771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,fp8,255,0.022602667411168415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,255,0.03138133386770884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,float16,511,0.0798880010843277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,fp8,511,0.02250666668017705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,511,0.04842666784922282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,float16,1,0.1341600020726522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,1,0.13513599832852682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,fp8,1,0.032357332607110344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,1,0.032730666299661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,float16,3,0.1344213287035624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,3,0.13424533605575562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,fp8,3,0.03249600032965342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,3,0.032373333970705666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,float16,15,0.13429866234461466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,float16,7,0.13405332962671915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,7,0.13512000441551208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,fp8,15,0.03252266595760981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,fp8,7,0.03243733445803324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,7,0.03266666581233343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,15,0.13432533542315164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,float16,31,0.13424000144004822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,15,0.032255999743938446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,31,0.13538666566212973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,fp8,31,0.03279466678698858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,31,0.03234666585922241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,float16,63,0.13589333494504294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,63,0.13422399759292603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,fp8,63,0.032144000132878624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,127,0.137061337629954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,63,0.03222399950027466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,float16,127,0.13710932930310568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,fp8,127,0.041322665909926094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,127,0.04125333329041799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,float16,255,0.14497066537539163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,255,0.14436266819636026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,fp8,255,0.04161066561937332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,255,0.05682133138179779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,float16,511,0.14538666605949402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,511,0.1828533411026001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,fp8,511,0.04144000013669332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,511,0.08703466256459554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,float16,1,0.25521600246429443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,1,0.25672000646591187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,fp8,1,0.06637866795063019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,1,0.06703466673692067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,float16,3,0.25573333104451496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,3,0.25733866294225055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,fp8,3,0.06607999900976817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,3,0.06669333577156067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,float16,7,0.25683732827504474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,7,0.25763734181722003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,fp8,7,0.06713066498438518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,7,0.06677333513895671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,float16,15,0.2561653256416321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,15,0.25762667258580524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,fp8,15,0.06575466692447662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,15,0.06668266654014587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,float16,31,0.260368009408315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,31,0.25887467463811237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,fp8,31,0.06694933275381725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,31,0.06671999891599019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,float16,63,0.26155734062194824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,63,0.26313066482543945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,fp8,63,0.06572266419728597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,63,0.0666133314371109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,float16,127,0.2581599950790405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,127,0.25809067487716675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,fp8,127,0.07865599791208903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,127,0.07826133569081624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,float16,255,0.27355732520421344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,fp8,255,0.07844266792138417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,255,0.2753760019938151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,255,0.10632000366846721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,fp8,511,0.0792799989382426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,float16,511,0.2759573260943095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,511,0.347925345102946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,511,0.1646719972292582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,float16,1,0.07534400125344594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,float16,1,0.07555200159549713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,fp8,1,0.02187199890613556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,fp8,1,0.02164799968401591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,float16,3,0.07572266459465027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,float16,3,0.07579733431339264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,fp8,3,0.02164799968401591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,fp8,3,0.021759999295075733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,float16,7,0.07526400188604991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,float16,7,0.07550399998823802
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,fp8,7,0.021930667261282604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,fp8,7,0.02165866643190384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,float16,15,0.07595733304818471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,float16,15,0.075573335091273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,fp8,15,0.02165866643190384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,fp8,15,0.021664001047611237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,float16,31,0.07540266712506612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,float16,31,0.07563733557860057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,fp8,31,0.02197333425283432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,fp8,31,0.02165333429972331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,float16,63,0.07425066828727722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,float16,63,0.07413333157698314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,fp8,63,0.02147199958562851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,fp8,63,0.021429332594076794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,float16,127,0.07689066727956136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,float16,127,0.07638933261235555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,fp8,127,0.026954665780067444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,fp8,127,0.026629333694775898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,float16,255,0.08293333152929942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,float16,255,0.0819946676492691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,fp8,255,0.026752000053723652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,fp8,255,0.040709334115187325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,float16,1,0.1361120045185089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,float16,1,0.1357493301232656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,fp8,1,0.03364266703526179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,fp8,1,0.03324266771475474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,float16,3,0.13686933120091757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,float16,3,0.13658666610717773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,fp8,3,0.03320533285538355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,fp8,3,0.03349333256483078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,float16,7,0.13616533080736795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,fp8,7,0.033717334270477295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,float16,7,0.13598400354385376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,fp8,7,0.03340800106525421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,float16,15,0.13675199945767721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,float16,15,0.13589866956075033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,fp8,15,0.03332799921433131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,fp8,15,0.03346133232116699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,float16,31,0.13734400272369385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,float16,31,0.13617066542307535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,fp8,31,0.03345066557327906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,fp8,31,0.03338133295377096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,float16,63,0.1373599966367086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,float16,63,0.13779733578364053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,fp8,63,0.0332640012105306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,fp8,63,0.033546666304270424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,float16,127,0.14008532961209616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,float16,127,0.14071466525395712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,fp8,127,0.04534400006135305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,fp8,127,0.044495999813079834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,float16,255,0.14986667037010193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,float16,255,0.14852266510327658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,fp8,255,0.04450133442878723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,fp8,255,0.059434667229652405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,float16,1,0.26263999938964844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,float16,1,0.2616426746050517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,fp8,1,0.06862399975458781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,fp8,1,0.06906133393446605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,float16,3,0.26155199607213336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,fp8,3,0.0688213308652242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,fp8,3,0.0689386675755183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,float16,7,0.2627679904301961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,float16,7,0.26150933901468915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,fp8,7,0.0682773341735204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,fp8,7,0.06903466582298279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,float16,15,0.26235733429590863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,float16,15,0.2627786596616109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,fp8,15,0.06897599995136261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,fp8,15,0.06945066650708516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,float16,31,0.265392005443573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,float16,31,0.26480533679326373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,fp8,31,0.06902933120727539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,fp8,31,0.06896533568700154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,float16,63,0.2680426637331645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,fp8,63,0.06938666601975758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,fp8,63,0.06941866874694824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,float16,127,0.2637919982274373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,fp8,127,0.07940799991289775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,fp8,127,0.07991999884446462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,float16,255,0.27914132674535114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,fp8,255,0.079434668024381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,fp8,255,0.10702932874361674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,float16,3,0.26124266783396405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,float16,1,0.5063573519388834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,float16,1,0.5049599806467692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,fp8,1,0.1307360033194224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,fp8,1,0.1306773324807485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,float16,3,0.5026079813639323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,fp8,3,0.13055466612180075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,float16,7,0.5050079822540283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,float16,63,0.2664586702982585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,float16,7,0.5030826727549235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,fp8,7,0.13056533535321554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,fp8,7,0.13059733311335245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,float16,255,0.28113067150115967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,float16,127,0.26368000109990436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,float16,15,0.5095413525899252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,float16,15,0.510101318359375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,fp8,15,0.13090667128562927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,fp8,15,0.13049599528312683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,fp8,31,0.13058132926623026
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,float16,31,0.5220479965209961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,float16,31,0.5179306666056315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,fp8,31,0.1302826702594757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,float16,63,0.5178666512171427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,float16,63,0.5188959836959839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,fp8,63,0.13038399815559387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,fp8,63,0.13103466232617697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,fp8,3,0.1321440041065216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,float16,127,0.5063680013020834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,fp8,127,0.1500320037206014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,float16,127,0.5070026715596517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,float16,3,0.5070773363113403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,1,0.008458666503429413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,fp8,127,0.14987199505170187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,1,0.008559999987483025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,1,0.0064213331788778305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,1,0.006624000146985054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,3,0.008527999743819237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,3,0.00847999999920527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,float16,255,0.5413386821746826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,3,0.006597333277265231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,fp8,255,0.15189866224924722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,float16,255,0.539242664972941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,fp8,255,0.20520534118016562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,3,0.006688000013430913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,7,0.008618666479984919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,7,0.00890666681031386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,7,0.006629333520929019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,7,0.006645333642760913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,15,0.009194666519761086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,15,0.008874666566650072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,15,0.006581333155433337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,15,0.006511999915043513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,31,0.009370666618148485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,31,0.009519999846816063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,31,0.006650666395823161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,31,0.007087999954819679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,127,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,127,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,63,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,63,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,63,0.006448000048597653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,63,0.0074560002734263735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,127,0.007941333577036858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,127,0.0074879998962084455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,255,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,255,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,255,0.008074666683872541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,255,0.008639999975760778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,511,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,511,0.01184533288081487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,511,0.007936000203092894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,511,0.009178666397929192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,1023,0.011584000041087469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,1023,0.011823999385039011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,1023,0.007946666950980822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,1023,0.00943999985853831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,2047,0.01681600014368693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,2047,0.01884799947341283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,2047,0.008266666904091835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,2047,0.011839999506870905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,4095,0.01718933383623759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,4095,0.021018666525681812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,4095,0.013738666971524557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,4095,0.008341333518425623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,8191,0.02083733429511388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,8191,0.028575999041398365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,8191,0.008538666491707167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,8191,0.01899733394384384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,16383,0.023578666150569916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,16383,0.04014399896065394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,16383,0.008432000254591307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,16383,0.02531733363866806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,32767,0.029631999631722767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,32767,0.062447999914487205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,32767,0.008634666601816813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,32767,0.04429866870244344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,1,0.00891733355820179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,1,0.009786666681369146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,1,0.005877333382765452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,1,0.005978666866819064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,3,0.008602666358153025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,3,0.00878399983048439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,65535,0.06833600004514058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,65535,0.03610666592915853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,7,0.008997333546479544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,3,0.005701333284378052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,3,0.006069333602984746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,65535,0.0955573320388794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,65535,0.008687999720374743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,7,0.009125333279371262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,7,0.005706666658322017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,15,0.008901333436369896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,31,0.009290666629870733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,7,0.006106666599710782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,15,0.008618666479984919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,15,0.006058666855096817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,15,0.005914666379491488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,31,0.009445333232482275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,31,0.005712000032265981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,31,0.006298666819930077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,63,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,63,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,63,0.005829333638151486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,63,0.006053333481152852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,255,0.011653333902359009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,127,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,127,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,127,0.006186666587988536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,127,0.007157333195209503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,255,0.011306667079528173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,255,0.007034666836261749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,255,0.007765333478649457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,511,0.011861333002646765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,1023,0.00730666642387708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,1023,0.00901333304742972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,511,0.011850666254758835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,511,0.00725333330531915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,511,0.007786666974425316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,1023,0.01624533285697301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,1023,0.016832000265518825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,2047,0.019306667149066925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,2047,0.023957334458827972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,2047,0.007962666451931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,2047,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,4095,0.023669332265853882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,4095,0.03244800120592117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,4095,0.00795199970404307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,8191,0.022682666778564453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,4095,0.014405333747466406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,8191,0.027749332288901012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,8191,0.04472533365090688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,8191,0.007706666365265846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,16383,0.033973333736260734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,16383,0.007903999959429106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,16383,0.07006399830182393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,16383,0.03143999973932902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,32767,0.0349386657277743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,32767,0.09270399808883667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,32767,0.008133333176374435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,32767,0.052815998593966164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,1,0.009861333295702934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,1,0.009962666779756546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,65535,0.03454400102297465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,1,0.005973333492875099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,65535,0.007877333089709282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,1,0.005888000130653381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,3,0.00996800015370051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,65535,0.14362133542696634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,3,0.009973333527644476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,3,0.005850666513045629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,3,0.005994666367769241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,7,0.009770666559537252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,65535,0.08635200063387553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,7,0.00983466642598311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,7,0.006351999938488007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,7,0.00730666642387708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,15,0.009839999799927076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,15,0.009770666559537252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,15,0.0059199997534354525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,15,0.0060159998635451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,31,0.009962666779756546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,31,0.01028266673286756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,31,0.007920000081261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,31,0.0059199997534354525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,63,0.00979200005531311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,63,0.009904000287254652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,63,0.005930666501323382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,63,0.0059199997534354525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,127,0.009935999910036722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,127,0.010469333579142889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,127,0.006186666587988536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,127,0.006133333469430606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,255,0.011968000481526056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,255,0.011776000261306763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,255,0.007135999699433644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,255,0.007530666887760162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,511,0.016522667060295742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,511,0.017551999539136887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,511,0.007311999797821045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,511,0.008383999889095625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,2047,0.008063999935984612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,1023,0.0215786670645078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,2047,0.016149333367745083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,1023,0.024288001159826916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,1023,0.007770666852593422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,1023,0.010474666953086853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,2047,0.027215999861558277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,2047,0.03642133375008901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,4095,0.03410666684309641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,4095,0.0525546669960022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,4095,0.008080000057816505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,4095,0.020618667205174763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,8191,0.033770665526390076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,8191,0.06733866532643636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,8191,0.007994666695594788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,8191,0.02956266701221466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,16383,0.03411199897527695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,16383,0.00821333316465219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,16383,0.08938666184743245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,16383,0.04643733302752177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,32767,0.03390933324893316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,32767,0.00820266641676426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,32767,0.13540800412495932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,32767,0.07910400132338206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,1,0.010239999741315842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,1,0.010186666622757912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,1,0.005968000118931134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,1,0.006111999973654747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,3,0.01044800008336703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,65535,0.22298665841420492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,3,0.010005333150426546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,65535,0.034101332227389015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,3,0.00595199999709924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,65535,0.008373333141207695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,65535,0.14311466614405313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,3,0.006170666466156642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,7,0.010496000448862711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,7,0.010053333515922228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,7,0.005957333371043205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,7,0.006165333092212677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,15,0.01009599988659223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,15,0.010239999741315842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,15,0.006133333469430606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,15,0.006144000217318535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,31,0.01044800008336703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,31,0.010122666756312052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,31,0.006074666976928711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,31,0.006224000205596288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,63,0.009893333539366722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,127,0.00996800015370051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,127,0.0063146669417619705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,63,0.010138666878143946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,63,0.006165333092212677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,63,0.006271999950210254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,127,0.01003200002014637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,127,0.006266666576266289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,255,0.017887999614079792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,255,0.01800000046690305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,255,0.00744000015159448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,255,0.008000000069538752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,1023,0.03298133363326391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,511,0.02295999974012375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,1023,0.01600533351302147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,511,0.024735999604066212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,511,0.008186666915814081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,511,0.009434666484594345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,1023,0.03953066716591517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,1023,0.008634666601816813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,2047,0.035504000882307686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,2047,0.050570666790008545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,2047,0.0084906667470932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,2047,0.019850666324297588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,4095,0.034927998979886375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,4095,0.06567466755708058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,4095,0.008496000121037165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,4095,0.028175999720891316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,8191,0.03499199946721395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,8191,0.08828799923261006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,8191,0.008394666636983553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,8191,0.04517866671085358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,16383,0.03486400097608566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,16383,0.13269866506258646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,16383,0.009050666665037474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,16383,0.0767626663049062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,32767,0.034645333886146545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,32767,0.22360533475875854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,32767,0.008565333361426989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,32767,0.14178666472434998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,1,0.014064000298579534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,1,0.007194666812817256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,3,0.013797332843144735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,65535,0.03526933242877325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,3,0.011407999942700068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,7,0.013989333063364029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,65535,0.39932799339294434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,65535,0.0086666668454806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,7,0.007109333450595538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,65535,0.2678080002466838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,15,0.015279999623696009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,15,0.007647999872763951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,31,0.01846933364868164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,31,0.007743999982873599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,63,0.018458666900793713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,63,0.007061333085099856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,127,0.007807999849319458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,127,0.018496000518401463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,255,0.013013333082199097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,255,0.009349333122372627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,511,0.01357866699496905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,511,0.00978133330742518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,1023,0.01937066639463107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,1023,0.01211200033624967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,2047,0.02847466617822647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,2047,0.015402667224407196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,4095,0.04282666742801666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,4095,0.025573333104451496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,8191,0.061109334230422974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,8191,0.0328053335348765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,16383,0.09124267101287842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,16383,0.056362668673197426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,1,0.010661333799362183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,1,0.006325333068768184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,3,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,32767,0.1336373289426168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,32767,0.08040533463160197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,3,0.006208000083764394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,7,0.010442666709423065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,7,0.006266666576266289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,15,0.010709332923094431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,15,0.0064213331788778305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,31,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,31,0.006106666599710782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,63,0.010591999938090643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,63,0.006346666564544042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,127,0.010442666709423065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,127,0.006800000245372455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,255,0.013616000612576803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,255,0.008863999818762144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,511,0.0195573332409064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,511,0.009472000102202097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,1023,0.02829333394765854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,1023,0.015573333948850632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,2047,0.04401599864164988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,2047,0.019189332922299702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,4095,0.0683786670366923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,4095,0.03012266755104065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,8191,0.0881760021050771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,8191,0.04357333481311798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,16383,0.13024000326792398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,16383,0.06751466790835063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,1,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,1,0.00808533343176047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,32767,0.2167146603266398
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,3,0.010645333677530289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,32767,0.11451199650764465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,3,0.006106666599710782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,7,0.010650667051474253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,7,0.006053333481152852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,15,0.011546666423479715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,15,0.005877333382765452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,31,0.010698666175206503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,31,0.006095999851822853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,63,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,63,0.0059199997534354525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,127,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,127,0.00761600024998188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,255,0.02109866589307785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,255,0.008506666868925095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,511,0.028223998844623566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,511,0.013728000223636627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,1023,0.04622933268547058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,1023,0.019973333925008774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,2047,0.025775998830795288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,2047,0.06665066878000896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,4095,0.08565866947174072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,4095,0.04022933294375738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,8191,0.1272053321202596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,8191,0.0641546646753947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,16383,0.21612266699473062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,16383,0.1113920013109843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,32767,0.2050293286641439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,32767,0.3951306740442912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,3,0.00720000018676122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,7,0.006586666529377301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,31,0.006464000170429547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,1,0.006581333155433337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,63,0.006490666419267654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,15,0.006810666372378667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,127,0.007370666911204656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,3,0.015578666081031164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,31,0.015461333096027374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,1,0.01551466683546702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,511,0.04753066599369049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,63,0.015311999867359797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,1023,0.06215466558933258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,1023,0.025237334271272022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,15,0.015311999867359797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,2047,0.08567999800046285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,2047,0.03938666731119156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,127,0.015605332950750986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,4095,0.12625599900881448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,4095,0.06355200211207072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,7,0.015184000134468079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,8191,0.21386667092641196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,255,0.030192000170548756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,8191,0.11068800091743469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,16383,0.389359990755717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,16383,0.2039466698964437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,1,0.008709333216150602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,1,0.006954666847983996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,3,0.008559999987483025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,32767,0.7412426471710205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,15,0.008698666468262672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,32767,0.39482665061950684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,3,0.00697066696981589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,7,0.00873066671192646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,7,0.0069866664707660675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,15,0.007189333438873291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,31,0.009343999748428663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,31,0.006917333230376244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,63,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,63,0.007146666447321574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,127,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,511,0.010048000141978264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,127,0.007813333223263422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,255,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,255,0.009103999783595404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,511,0.01532799998919169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,255,0.013178666432698568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,1023,0.017082666357358296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,1023,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,2047,0.01646399994691213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,2047,0.011850666254758835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,511,0.01922133316596349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,4095,0.02719466636578242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,4095,0.013429333766301474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,8191,0.018138666947682697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,8191,0.016336000214020412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,16383,0.019626667102177937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,16383,0.018687999496857326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,32767,0.024101334313551586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,32767,0.023898666103680927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,65535,0.02863466739654541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,3,0.006048000107208888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,65535,0.029872000217437744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,1,0.00980266680320104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,1,0.00589866687854131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,3,0.008629333227872849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,7,0.008565333361426989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,131071,0.03949866692225138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,131071,0.039808000127474465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,7,0.005893333504597346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,15,0.009039999917149544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,127,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,15,0.0058453331391016645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,31,0.009359999870260557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,31,0.006042666733264923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,63,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,63,0.005984000240763028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,127,0.0069226666043202085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,1023,0.008512000242869059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,255,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,255,0.00766933336853981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,511,0.015423999478419622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,511,0.008122666428486506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,1023,0.016458666572968166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,2047,0.01676799977819125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,2047,0.009509333098928133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,4095,0.015130666395028433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,4095,0.010640000303586325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,8191,0.018181333939234417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,8191,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,16383,0.019802667200565338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,16383,0.015125333021084467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,32767,0.02470933397610982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,32767,0.019088000059127808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,1,0.00854399986565113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,65535,0.03029866764942805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,65535,0.023770667612552643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,1,0.005770666524767876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,3,0.008992000172535578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,3,0.005706666658322017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,7,0.008656000097592672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,7,0.005695999910434087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,131071,0.059418668349583946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,131071,0.03551999976237615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,15,0.008767999708652496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,15,0.005658666913708051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,255,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,31,0.009882666791478792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,31,0.005674666414658229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,63,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,63,0.0058186668902635574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,127,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,127,0.006271999950210254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,2047,0.014378666877746582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,255,0.007418666655818622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,511,0.015360000232855478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,8191,0.018853332847356796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,511,0.008192000289758047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,1023,0.016421332955360413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,1023,0.008352000266313553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,2047,0.009237333511312803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,4095,0.015450666348139444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,4095,0.01033599985142549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,8191,0.012293333808581034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,16383,0.02109866589307785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,16383,0.01545599972208341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,32767,0.03490666548411051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,32767,0.020645332833131153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,131071,0.0831573357184728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,65535,0.04890666902065277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,65535,0.02604266752799352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,1,0.008757333581646284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,1,0.005999999741713206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,3,0.008559999987483025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,131071,0.04114133367935816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,3,0.005946666623155276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,7,0.008639999975760778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,15,0.008949333180983862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,15,0.00590933362642924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,31,0.009786666681369146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,31,0.005717333406209946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,255,0.00785600021481514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,63,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,63,0.00596266674498717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,127,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,127,0.006186666587988536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,255,0.011488000551859537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,511,0.015333333363135656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,511,0.008127999802430471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,1023,0.013434667140245438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,1023,0.008181333541870117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,2047,0.014917333920796713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,7,0.0061919999619325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,2047,0.009594666461149851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,4095,0.01646399994691213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,4095,0.010602666685978571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,8191,0.02566933383544286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,16383,0.030879999200503033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,16383,0.016906666258970898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,65535,0.07886933286984761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,32767,0.05050133168697357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,32767,0.024288001159826916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,131071,0.13640000422795615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,3,0.009114666531483332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,65535,0.05198400219281515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,1,0.008832000195980072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,1,0.0069440001000960665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,131071,0.06731200218200684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,15,0.007055999711155891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,3,0.007178666690985362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,8191,0.013818666338920593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,7,0.008799999952316284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,7,0.0069386667261521024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,15,0.008976000050703684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,31,0.009535999968647957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,31,0.007109333450595538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,63,0.011551999797423681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,63,0.00730666642387708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,127,0.011482667177915573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,255,0.011445333560307821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,127,0.007690666864315669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,255,0.009514666472872099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,2047,0.011760000139474869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,511,0.015552000453074774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,511,0.009679999823371569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,1023,0.016650666793187458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,1023,0.01071999967098236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,2047,0.02752533306678136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,4095,0.0161920003592968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,4095,0.01340266689658165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,8191,0.019013332823912304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,8191,0.015802666544914246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,16383,0.021946666141351063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,16383,0.018965333700180054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,32767,0.025461333493391674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,32767,0.024090667565663654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,65535,0.031914666295051575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,65535,0.03028800090154012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,1,0.008821333448092142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,7,0.008976000050703684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,1,0.00921066664159298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,131071,0.06159999966621399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,3,0.008725333337982496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,131071,0.04607999821503957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,3,0.006026666611433029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,31,0.0060159998635451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,7,0.0061919999619325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,15,0.009072000160813332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,15,0.00613866684337457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,31,0.009712000067035357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,63,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,63,0.006213333457708359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,127,0.01128000020980835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,127,0.006453333422541618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,255,0.011365332951148352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,255,0.007920000081261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,511,0.015717333803574245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,511,0.008410666758815447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,1023,0.016613333175579708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,1023,0.00879466657837232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,8191,0.013034666577974955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,2047,0.015253332753976187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,2047,0.009679999823371569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,4095,0.015872000406185787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,4095,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,8191,0.020080000162124634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,16383,0.021727999051411945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,16383,0.015237333873907724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,32767,0.035674666364987694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,32767,0.021781332790851593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,65535,0.050848002235094704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,65535,0.027072000006834667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,1,0.008687999720374743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,131071,0.08447999755541484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,131071,0.042453333735466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,1,0.005744000275929769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,3,0.008997333546479544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,3,0.006309333567818006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,31,0.0057920000205437345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,7,0.008736000085870424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,7,0.0058613332609335584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,15,0.008853333070874214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,15,0.005717333406209946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,31,0.00979200005531311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,255,0.007631999750932057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,63,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,63,0.0058026667684316635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,127,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,1023,0.008357333640257517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,127,0.006309333567818006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,255,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,511,0.015482666591803232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,511,0.008170666793982187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,1023,0.01351999988158544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,2047,0.015509333461523056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,2047,0.00921066664159298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,4095,0.016623999923467636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,4095,0.01062400018175443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,8191,0.025685332715511322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,8191,0.014064000298579534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,16383,0.03190933416287104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,16383,0.01684800038735072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,32767,0.05124799907207489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,32767,0.024447999894618988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,65535,0.08101866642634074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,65535,0.04956800242265066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,131071,0.13717333475748697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,1,0.008821333448092142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,131071,0.06753066678841908
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,1,0.005829333638151486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,3,0.008757333581646284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,3,0.005957333371043205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,31,0.0058186668902635574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,7,0.008752000207702318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,7,0.005872000008821487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,15,0.009018666421373686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,15,0.0058399997651577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,31,0.00966933307548364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,63,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,63,0.006111999973654747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,127,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,127,0.00625599982837836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,255,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,255,0.007685333490371704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,511,0.012629333883523941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,511,0.007925333455204964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,1023,0.013338666409254074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,1023,0.008634666601816813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,8191,0.03485333422819773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,2047,0.020282667130231857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,2047,0.010133333504199982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,4095,0.02369066576162974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,4095,0.011802667131026586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,8191,0.015919999529918034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,65535,0.07121600210666656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,16383,0.053861334919929504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,16383,0.027984000742435455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,32767,0.08077866832415263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,32767,0.04030933231115341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,65535,0.13698133826255798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,131071,0.24023999770482382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,1,0.010458666831254959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,1,0.007189333438873291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,3,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,131071,0.11570133765538533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,3,0.00702400008837382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,7,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,7,0.007226666435599327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,15,0.010703999549150467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,15,0.007477333148320516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,31,0.010608000059922537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,127,0.007663999994595845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,31,0.007770666852593422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,63,0.01044800008336703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,63,0.0069919998447100324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,127,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,255,0.013573333621025085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,255,0.009535999968647957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,511,0.01959466685851415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,511,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,1023,0.028853334486484528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,4095,0.03938666731119156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,1023,0.019194666296243668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,2047,0.04448533554871877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,16383,0.1321440041065216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,1,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,2047,0.02365333338578542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,3,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,4095,0.06941333413124084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,8191,0.08896000186602275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,8191,0.053898667295773826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,16383,0.07999466856320699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,1,0.006181333214044571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,3,0.006501333167155583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,7,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,7,0.006458666796485583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,15,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,15,0.006229333579540253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,31,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,31,0.006271999950210254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,63,0.010656000425418219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,63,0.006602666651209195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,127,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,127,0.00730666642387708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,255,0.02124800036350886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,255,0.009306666751702627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,511,0.02826666583617528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,511,0.015274666249752045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,4095,0.08782399694124858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,1023,0.04598399996757507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,1023,0.022170667846997578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,2047,0.06674133241176605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,2047,0.028581333657105763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,4095,0.042810668547948204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,8191,0.12865599989891052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,8191,0.06713599960009257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,16383,0.21941866477330527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,3,0.006469333544373512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,1,0.015583999454975128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,15,0.015583999454975128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,16383,0.11316800117492676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,1,0.006442666674653689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,3,0.015552000453074774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,7,0.015413332730531693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,7,0.00655466690659523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,15,0.006415999804933866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,31,0.015573333948850632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,31,0.006730666384100914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,63,0.01525866612792015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,63,0.006645333642760913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,127,0.015706667055686314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,127,0.0074879998962084455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,255,0.03030933439731598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,255,0.013749333719412485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,511,0.04722133278846741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,511,0.019695999721686046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,1023,0.0634080022573471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,1023,0.02589866767326991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,2047,0.08636266986529033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,2047,0.040048000713189445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,4095,0.127893328666687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,4095,0.0646666685740153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,1,0.02508266766866048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,8191,0.21753066778182983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,8191,0.11161599556605022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,16383,0.3970986604690552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,1,0.009642666826645533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,16383,0.2056480050086975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,3,0.025125332176685333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,3,0.009850666547815004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,7,0.025072000920772552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,7,0.009653333574533463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,15,0.02489600082238515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,15,0.01007466639081637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,31,0.024864000578721363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,31,0.009877333417534828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,63,0.024911999702453613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,63,0.009813333551088968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,127,0.02517866591612498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,127,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,255,0.04987200101216634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,255,0.01937066639463107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,511,0.06444799900054932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,511,0.025301332275072735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,1023,0.08348266283671062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,1023,0.03995733211437861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,2047,0.1301759978135427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,2047,0.06382933259010315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,4095,0.2199359933535258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,4095,0.1113973359266917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,8191,0.20594133933385214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,8191,0.39684800306955975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,16383,0.7513279914855957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,1,0.008682666967312494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,16383,0.39638400077819824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,1,0.00684799998998642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,3,0.008762666955590248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,3,0.007237333183487256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,7,0.008938666433095932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,7,0.006927999978264173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,15,0.00902399979531765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,15,0.006954666847983996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,31,0.009722666814923286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,127,0.011407999942700068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,31,0.006831999868154526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,63,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,63,0.0075040000180403394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,127,0.007813333223263422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,255,0.011525332927703857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,255,0.00984533317387104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,511,0.01580799991885821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,511,0.010117333382368088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,1023,0.027274665733178455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,1023,0.010575999816258749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,2047,0.01658133293191592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,2047,0.011711999773979187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,4095,0.016965333372354507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,4095,0.013343999783198038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,8191,0.020597333709398907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,8191,0.01602666700879733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,16383,0.02317333221435547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,16383,0.018986667195955913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,32767,0.03807999938726425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,32767,0.0273333340883255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,1,0.008645333349704742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,65535,0.053557331363360085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,65535,0.03455466777086258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,1,0.006165333092212677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,3,0.010341333225369453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,3,0.006095999851822853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,7,0.008682666967312494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,7,0.006133333469430606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,131071,0.08708799878756206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,131071,0.05412266651789347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,15,0.008922666932145754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,15,0.006154666965206464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,31,0.010010666524370512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,31,0.006277333324154218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,63,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,63,0.006154666965206464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,127,0.012432000289360682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,127,0.006720000257094701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,255,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,255,0.008042666440208754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,511,0.01609066625436147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,511,0.008234666660428047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,1023,0.013418667018413544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,1023,0.00871999996403853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,2047,0.016389333953460056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,2047,0.00973866693675518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,4095,0.01883200059334437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,4095,0.011440000186363855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,8191,0.026704000929991405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,8191,0.014869333555301031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,16383,0.03259733319282532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,16383,0.018021332720915478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,32767,0.05309333403905233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,32767,0.02513599892457326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,65535,0.05267733335494995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,65535,0.0824480007092158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,1,0.008778666456540426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,131071,0.1377066671848297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,1,0.00578666664659977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,3,0.008805333326260248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,3,0.0058613332609335584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,7,0.009482666850090027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,131071,0.07087466617425282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,7,0.006074666976928711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,15,0.008986666798591614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,15,0.006058666855096817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,31,0.009637333452701569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,31,0.006021333237489064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,63,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,63,0.0058133335163195925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,127,0.011322667201360067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,127,0.0064266665528217954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,255,0.011461333682139715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,255,0.007733333234985669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,511,0.012837332983811697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,511,0.007920000081261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,1023,0.013327999661366144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,1023,0.008623999853928884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,2047,0.0215786670645078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,2047,0.010506667196750641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,4095,0.024501333634058636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,4095,0.011882666498422623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,8191,0.035717333356539406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,8191,0.016234666109085083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,16383,0.055173332492510475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,16383,0.027386667827765148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,32767,0.08255999783674876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,32767,0.04045333216587702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,65535,0.07121600210666656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,65535,0.1368053356806437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,131071,0.2409493327140808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,1,0.009008000294367472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,1,0.00590933362642924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,3,0.00877333308259646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,3,0.0069226666043202085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,7,0.00877333308259646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,7,0.006101333225766818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,131071,0.1153546671072642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,15,0.009136000027259191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,15,0.005829333638151486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,31,0.009690666571259499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,127,0.006453333422541618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,31,0.006053333481152852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,63,0.011482667177915573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,63,0.0060159998635451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,127,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,255,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,255,0.007418666655818622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,511,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,511,0.009136000027259191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,1023,0.01926933353145917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,1023,0.009397333487868309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,2047,0.028255999088287354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,2047,0.011989332735538483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,4095,0.04053866614898046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,4095,0.01830400029818217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,8191,0.05892266829808553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,8191,0.02386133372783661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,16383,0.08874133229255676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,16383,0.044250667095184326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,32767,0.13129599889119467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,32767,0.06523733337720235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,65535,0.22472532590230307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,65535,0.11065600315729777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,1,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,1,0.0074560002734263735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,131071,0.4185493389765422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,3,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,131071,0.2026080091794332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,3,0.007541333635648091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,7,0.011328000575304031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,7,0.007184000064929326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,15,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,15,0.007536000261704127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,31,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,31,0.007370666911204656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,63,0.0075040000180403394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,63,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,127,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,511,0.018757333358128864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,127,0.008442666381597519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,255,0.02144533395767212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,255,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,511,0.02888533224662145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,1023,0.048010667165120445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,1023,0.02752533306678136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,2047,0.06735999882221222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,2047,0.038319999972979225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,4095,0.08826133608818054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,4095,0.05169066786766052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,8191,0.13103999694188437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,1,0.01590399940808614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,8191,0.07778133451938629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,1,0.007120000198483467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,3,0.015674666812022526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,3,0.006751999879876773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,7,0.016010666886965435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,7,0.007216000308593114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,15,0.015562667200962702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,15,0.00696000022192796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,31,0.016021333634853363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,31,0.006842666616042455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,63,0.015605332950750986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,63,0.006890666360656421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,127,0.01603200038274129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,127,0.008325333396593729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,255,0.030853333572546642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,255,0.015317333241303762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,511,0.04780266682306925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,511,0.02181866765022278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,1023,0.06233066817124685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,1023,0.0286613330245018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,2047,0.08591467142105103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,2047,0.042917331059773765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,4095,0.12966932853062949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,4095,0.06740266581376393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,1,0.02590399980545044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,1,0.009568000212311745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,8191,0.22223466634750366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,3,0.025306666890780132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,8191,0.11733333269755046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,3,0.010277333358923594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,7,0.02513599892457326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,7,0.00973866693675518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,15,0.025274666647116344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,15,0.009717333440979322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,31,0.025253333151340485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,31,0.009898666913310686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,63,0.025194667279720306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,63,0.010064000263810158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,127,0.025173333783944447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,127,0.011445333560307821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,255,0.05153066913286845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,255,0.01961600035429001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,511,0.06481066842873891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,511,0.025968000292778015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,1023,0.08408000071843465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,1023,0.04070399949947993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,2047,0.13218133648236594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,4095,0.22566932439804077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,2047,0.0651093324025472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,4095,0.11287466684977214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,1,0.04461866617202759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,1,0.014149333039919535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,3,0.013786666095256805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,8191,0.4044320185979207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,8191,0.20710933208465576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,3,0.043925335009892784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,7,0.04364266494909922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,7,0.013738666971524557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,15,0.04487466812133789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,15,0.0141546664138635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,31,0.04418666660785675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,31,0.013818666338920593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,63,0.04331199824810028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,255,0.021722666919231415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,63,0.013712000101804733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,127,0.043968002001444496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,1023,0.11851732929547627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,127,0.015557333827018738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,255,0.053871999184290566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,511,0.07354666789372762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,511,0.03722133239110311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,1023,0.061205332477887474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,2047,0.21290133396784464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,2047,0.10886399944623311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,4095,0.3932906786600749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,4095,0.20563733577728271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,1,0.01632000009218852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,8191,0.39345065752665204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,8191,0.7498559951782227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,1,0.00821333316465219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,3,0.01621866722901662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,3,0.00810666692753633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,7,0.016229332735141117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,7,0.007946666950980822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,15,0.01632533346613248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,15,0.008005333443482717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,31,0.01692266638080279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,31,0.008826666822036108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,63,0.01588800052801768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,63,0.008047999814152718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,127,0.016255999604860943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,255,0.018543999642133713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,127,0.00897066667675972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,255,0.03161599983771642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,511,0.04964800179004669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,511,0.026352000733216602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,1023,0.06404266754786174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,1023,0.037733333806196846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,2047,0.08833066622416179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,1,0.026165333886941273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,1,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,2047,0.05264533559481303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,3,0.025759999950726826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,3,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,7,0.02586666742960612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,7,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,15,0.025749333202838898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,15,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,31,0.02619733413060506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,31,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,63,0.025663999219735462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,63,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,127,0.0260959987839063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,127,0.013056000073750814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,255,0.051226665576299034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,255,0.021962667504946392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,1023,0.04372799893220266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,511,0.06578666468461354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,511,0.02908266584078471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,1023,0.08471999565760295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,1,0.04386133452256521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,2047,0.06763733426729839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,2047,0.1330933372179667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,1,0.013957332819700241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,3,0.043951998154322304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,3,0.014021333307027817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,7,0.0440586656332016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,7,0.013663999736309052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,63,0.013936000565687815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,15,0.043935999274253845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,15,0.01393066719174385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,31,0.04484266539414724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,31,0.014015999933083853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,63,0.04346133271853129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,127,0.04474666714668274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,127,0.016255999604860943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,255,0.05421866476535797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,255,0.02333866556485494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,511,0.07439466814200084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,511,0.03775466730197271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,1023,0.11983999609947205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,1023,0.06121066709359487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,2047,0.215664009253184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,2047,0.1090186635653178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,1,0.08038933575153351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,1,0.023258666197458904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,3,0.08040533463160197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,3,0.02367466688156128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,7,0.08072000245253245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,7,0.02342933416366577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,15,0.08032533526420593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,15,0.023408000667889912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,31,0.08017066617806752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,31,0.023290666441122692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,63,0.07975466549396515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,63,0.02349333216746648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,127,0.08188266555468242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,511,0.1301813324292501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,127,0.02881066749493281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,255,0.09604266285896301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,255,0.04113066693147024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,511,0.06457599997520447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,1023,0.21500800053278604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,1023,0.1107413371404012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,2047,0.40374934673309326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,1,0.026661333938439686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,2047,0.20380266507466635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,1,0.012725333372751871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,3,0.026554666459560394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,3,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,7,0.026320000489552815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,7,0.012863999853531519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,15,0.026677332818508148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,15,0.012794667234023413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,31,0.027082666754722595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,31,0.012821332861979803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,63,0.026399999856948853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,63,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,127,0.026373334228992462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,127,0.015290666371583939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,255,0.0536053329706192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,255,0.02647999922434489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,511,0.06784533460934956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,511,0.03867733230193456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,1023,0.08758933345476787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,1023,0.052853330969810486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,1,0.045370668172836304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,1,0.015333333363135656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,3,0.04568533102671305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,15,0.04543466866016388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,3,0.015392000476519266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,7,0.04637866715590159
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,31,0.015450666348139444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,7,0.015184000134468079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,15,0.01526933287580808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,63,0.04525866607824961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,31,0.04586133360862732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,63,0.01526933287580808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,255,0.05604266623655955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,127,0.04725866516431173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,127,0.01915733392039935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,255,0.02590399980545044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,511,0.07698666552702586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,511,0.040565334260463715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,1023,0.12174399693806966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,1023,0.06440000236034393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,1,0.0806933343410492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,1,0.02367999901374181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,3,0.08142399787902832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,3,0.023631999890009563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,7,0.0811359981695811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,7,0.023584000766277313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,15,0.0809333324432373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,15,0.024538666009902954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,31,0.08083733419577281
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,31,0.023786666492621105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,127,0.08323733508586884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,63,0.08172266681989034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,63,0.023423999547958374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,127,0.030837332208951313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,255,0.09693866968154907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,255,0.04268266757329305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,511,0.13269333044687906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,511,0.06614933411280315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,1023,0.21694932381312051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,1023,0.11121066411336263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,1,0.1495733360449473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,1,0.046122665206591286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,3,0.14986133575439453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,3,0.045754666129748024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,7,0.14965333541234335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,7,0.0467199981212616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,15,0.15025599797566733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,15,0.04596266647179922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,127,0.1513653298219045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,31,0.1513866682847341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,31,0.046154667933781944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,63,0.15219199657440186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,63,0.045594667394955955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,127,0.05444799860318502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,1023,0.1995519995689392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,255,0.17804267009099325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,255,0.07369600236415863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,511,0.24408000707626343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,511,0.11449600259462993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,1023,0.4023199876149495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,1,0.009066666786869368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,1,0.007338666667540868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,3,0.009056000038981438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,3,0.00690133310854435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,7,0.009119999905427298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,7,0.007045333584149678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,15,0.009109333157539368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,15,0.0069440001000960665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,31,0.009824000298976898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,31,0.007258666679263115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,63,0.011440000186363855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,63,0.006895999734600385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,127,0.01157333329319954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,127,0.007541333635648091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,255,0.011610666910807291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,255,0.009317333499590555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,511,0.025813333690166473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,511,0.00997866690158844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,1023,0.013525333255529404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,1023,0.010485333700974783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,2047,0.01717866708834966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,2047,0.011813333878914515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,4095,0.017893332988023758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,4095,0.013530666629473368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,8191,0.027248000105222065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,8191,0.018677332748969395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,16383,0.033157333731651306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,16383,0.022128000855445862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,32767,0.05629333357016245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,32767,0.03212266663710276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,65535,0.08307733138402303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,1,0.006058666855096817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,65535,0.06305066744486491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,131071,0.13843199610710144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,1,0.009077333534757296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,131071,0.08372799555460612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,3,0.008933333059151968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,3,0.005973333492875099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,7,0.00874133345981439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,7,0.005999999741713206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,15,0.009039999917149544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,15,0.00620266670982043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,127,0.011322667201360067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,127,0.00679466687142849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,31,0.009888000165422758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,31,0.0063146669417619705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,63,0.011424000064531961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,63,0.006085333103934924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,255,0.011717333147923151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,255,0.007797333101431529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,511,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,511,0.008197333042820295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,1023,0.013445333888133367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,1023,0.008997333546479544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,4095,0.013077333569526672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,2047,0.021642667551835377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,2047,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,4095,0.02473066747188568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,8191,0.03694933404525121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,8191,0.017450666675964992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,16383,0.05548266569773356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,16383,0.02887466549873352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,32767,0.08370666702588399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,32767,0.042677332957585655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,65535,0.13678399721781412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,65535,0.07321066657702129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,1,0.009045333291093508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,1,0.005669333040714264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,3,0.008997333546479544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,131071,0.24011200666427612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,3,0.00596266674498717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,131071,0.11700266599655151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,7,0.008997333546479544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,7,0.005765333150823911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,15,0.009226666763424873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,15,0.005882666756709416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,31,0.009962666779756546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,31,0.005914666379491488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,63,0.011525332927703857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,63,0.006704000135262807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,127,0.011477333803971609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,1023,0.019146667172511418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,127,0.006213333457708359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,255,0.012725333372751871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,255,0.00761600024998188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,511,0.013151999562978745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,511,0.0080960001796484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,1023,0.009632000078757605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,2047,0.028213332096735638
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,2047,0.011855999628702799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,4095,0.040336000422636666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,4095,0.01852799952030182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,8191,0.05921066800753275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,8191,0.02480533222357432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,16383,0.08922132849693298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,16383,0.043098668257395424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,32767,0.13211199641227722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,32767,0.06518400212128957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,65535,0.22546666860580444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,65535,0.11106133460998535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,131071,0.42188799381256104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,1,0.0102613332370917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,131071,0.20286933581034342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,1,0.00600533311565717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,3,0.01051733394463857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,3,0.005904000252485275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,7,0.010197333370645842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,7,0.006064000229040782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,15,0.010464000205198923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,15,0.005904000252485275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,31,0.010378666842977205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,31,0.006037333359320958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,63,0.010469333579142889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,63,0.005914666379491488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,127,0.01028266673286756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,127,0.006309333567818006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,255,0.013445333888133367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,255,0.007727999861041705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,511,0.019445333629846573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,511,0.009088000282645226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,1023,0.028362666567166645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,1023,0.013728000223636627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,2047,0.04307200014591217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,2047,0.0173333336909612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,4095,0.06855999926726024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,4095,0.026101333399613697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,8191,0.0874079962571462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,8191,0.040607998768488564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,16383,0.127893328666687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,16383,0.0639519989490509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,32767,0.21660800774892172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,32767,0.11237333218256633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,65535,0.3928266763687134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,65535,0.206879993279775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,1,0.04688533147176107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,131071,0.7496373653411865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,131071,0.3910026550292969
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,1,0.01833600054184596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,3,0.04668800036112467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,3,0.01868266612291336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,7,0.04677333434422811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,7,0.01855466639002164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,15,0.046800002455711365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,15,0.0184906671444575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,31,0.04747733473777771
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,31,0.018405333161354065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,63,0.04624533156553904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,63,0.018346666047970455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,127,0.049173335234324135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,127,0.023221333821614582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,255,0.05901333192984263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,255,0.03568000098069509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,511,0.07901333272457123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,511,0.04879466692606608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,1,0.08310933411121368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,1,0.026309333741664886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,3,0.02626666675011317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,3,0.08355733752250671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,7,0.08337600032488506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,7,0.026928000152111053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,15,0.08273066580295563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,15,0.026288000245889027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,31,0.0830026666323344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,31,0.02628266563018163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,63,0.08408533533414204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,63,0.02672533442576726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,127,0.0863200028737386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,127,0.03741333385308584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,255,0.10000532865524292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,255,0.049072002371152244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,511,0.13727999726931253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,511,0.07044266661008199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,1,0.15054933230082193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,1,0.04869333406289419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,3,0.1516800026098887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,3,0.04870399832725525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,7,0.1508693297704061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,7,0.04863999783992767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,15,0.15180266896883646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,15,0.04860266546408335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,31,0.15386133392651877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,31,0.04897066454092661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,63,0.15506133437156677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,63,0.048623998959859215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,127,0.15410666664441428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,127,0.056943997740745544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,255,0.18143999576568604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,255,0.0761599987745285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,511,0.2488159934679667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,511,0.11771733562151591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,3,0.08841066559155782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,1,0.2922826608022054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,1,0.08915199836095174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,3,0.2900480031967163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,7,0.292303999265035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,7,0.08864532907803853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,15,0.2940053343772888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,15,0.08915199836095174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,31,0.29901333649953205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,31,0.0888266662756602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,63,0.29658667246500653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,63,0.08844266335169475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,127,0.29469333092371625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,127,0.1011786659558614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,255,0.3427040179570516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,255,0.1379039982954661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,511,0.47139732042948407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,511,0.2160586714744568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,1,0.08612799644470215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,fp8,1,0.03559466699759165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,3,0.08613866567611694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,fp8,3,0.03519999980926514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,7,0.08572266499201457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,fp8,7,0.035642666121323906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,15,0.08633066217104594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,fp8,15,0.03659199923276901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,31,0.08749333024024963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,fp8,31,0.03550933301448822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,63,0.0886240005493164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,fp8,63,0.03497066597143809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,127,0.08948799967765808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,fp8,127,0.05566399792830149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,255,0.1053493320941925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,fp8,255,0.06425599753856659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,1,0.15573333700497946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,fp8,1,0.05769066512584686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,3,0.15653333067893982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,fp8,3,0.057914664347966514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,fp8,31,0.05755733450253805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,7,0.15602133671442667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,fp8,63,0.05750399827957153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,fp8,7,0.05788266658782959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,15,0.15796266992886862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,fp8,15,0.057855998476346336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,31,0.16014933586120605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,63,0.16040000319480896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,127,0.16087999939918518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,fp8,127,0.067930668592453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,255,0.1876159906387329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,fp8,255,0.0872213343779246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,fp8,1,0.09111467003822327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,1,0.29313067595163983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,3,0.2948053280512492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,fp8,15,0.09128000338872273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,fp8,3,0.09140800436337788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,7,0.2958293358484904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,fp8,7,0.09108266234397888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,15,0.2986453374226888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,127,0.29758399724960327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,31,0.30145599444707233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,fp8,31,0.09110933542251587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,63,0.301744004090627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,fp8,63,0.09131733576456706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,fp8,127,0.10707199573516846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,fp8,3,0.1710240046183268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,255,0.34862399101257324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,fp8,255,0.14298133055369058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,1,0.5740319887797037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,fp8,1,0.17034133275349936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,3,0.57697065671285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,7,0.5846346616744995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,fp8,7,0.17178666591644287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,15,0.5907466808954874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,fp8,15,0.17068799336751303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,31,0.5871253410975138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,fp8,31,0.17015999555587769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,63,0.5802719990412394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,fp8,63,0.17062934239705405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,255,0.672538677851359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,127,0.5781333446502686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,fp8,127,0.1952106753985087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,1,0.008682666967312494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,fp8,255,0.26471465826034546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,1,0.0069333333522081375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,3,0.008703999842206636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,3,0.007311999797821045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,7,0.008752000207702318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,7,0.00706666645904382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,15,0.009370666618148485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,15,0.007354666789372762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,31,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,31,0.0069440001000960665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,63,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,63,0.007280000175038974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,127,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,1023,0.013354666531085968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,127,0.007770666852593422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,255,0.012944000462690989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,255,0.009354666496316591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,511,0.013162666310866674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,511,0.009632000078757605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,1023,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,2047,0.021712000171343487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,2047,0.013178666432698568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,4095,0.02573866645495097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,4095,0.015562667200962702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,16383,0.03686933219432831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,8191,0.03742400060097376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,8191,0.021151999632517498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,16383,0.05682666599750519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,32767,0.08406399687131245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,32767,0.05193066596984863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,65535,0.1386560002962748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,65535,0.08803199728329976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,1,0.008623999853928884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,1,0.006197333335876465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,3,0.008912000184257826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,3,0.006053333481152852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,7,0.008592000231146812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,7,0.006346666564544042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,15,0.00927466650803884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,15,0.006277333324154218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,31,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,31,0.006309333567818006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,63,0.011306667079528173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,63,0.005935999875267346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,127,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,127,0.006655999769767125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,255,0.012432000289360682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,255,0.008058666562040647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,2047,0.028079998989899952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,511,0.013434667140245438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,511,0.008469333251317343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,1023,0.019274666905403137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,1023,0.010250666489203772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,2047,0.012810666114091873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,4095,0.04098666707674662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,4095,0.019925333559513092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,8191,0.058933332562446594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,8191,0.0262773334980011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,16383,0.08945066730181377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,16383,0.04567466676235199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,32767,0.13008000453313193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,32767,0.06817600131034851
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,65535,0.22343466679255167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,65535,0.11371733744939168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,1,0.010144000252087912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,1,0.006282666698098183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,3,0.010437333335479101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,3,0.005941333249211311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,7,0.010154666379094124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,7,0.005824000264207522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,15,0.010165333126982054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,15,0.005978666866819064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,31,0.010090666512648264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,31,0.0063680000603199005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,63,0.010378666842977205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,255,0.013317332913478216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,63,0.005775999898711841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,127,0.010202666744589806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,127,0.006346666564544042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,255,0.007861333588759104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,511,0.01904533306757609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,511,0.009152000149091085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,1023,0.027930667002995808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,1023,0.013951999445756277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,2047,0.04260799785455068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,2047,0.01738133281469345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,4095,0.06738133231798808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,4095,0.0266239990790685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,8191,0.08668800195058186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,8191,0.04098666707674662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,16383,0.12804800271987915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,16383,0.06483200192451477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,32767,0.21625600258509317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,32767,0.11160000165303548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,65535,0.39343468348185223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,1,0.010549332946538925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,65535,0.20721600453058878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,1,0.006181333214044571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,3,0.0106133334338665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,3,0.00596266674498717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,7,0.010634666929642359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,7,0.00589866687854131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,15,0.010458666831254959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,15,0.006042666733264923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,31,0.010538666198650995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,31,0.006128000095486641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,63,0.0106133334338665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,63,0.005941333249211311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,127,0.010666667173306147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,127,0.006448000048597653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,255,0.020746666938066483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,255,0.008623999853928884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,511,0.028421332438786823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,511,0.01320533330241839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,1023,0.046165332198143005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,1023,0.019274666905403137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,2047,0.065461332599322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,2047,0.025173333783944447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,4095,0.08593066533406575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,4095,0.03969600051641464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,8191,0.1258240044116974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,8191,0.06363200147946675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,16383,0.2127359906832377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,16383,0.11003733674685161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,32767,0.3890186548233032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,32767,0.2042186657587687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,1,0.008709333216150602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,1,0.00871999996403853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,1,0.00600533311565717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,65535,0.7392533620198568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,65535,0.3933173418045044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,1,0.006064000229040782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,3,0.008645333349704742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,3,0.008672000219424566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,3,0.006159999718268712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,3,0.006159999718268712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,7,0.008762666955590248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,7,0.008799999952316284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,7,0.005957333371043205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,7,0.006048000107208888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,15,0.008821333448092142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,15,0.008842666943868002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,15,0.006122666721542676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,15,0.00624533308049043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,31,0.009450666606426239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,31,0.009488000224033991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,31,0.006053333481152852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,31,0.0069386667261521024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,63,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,63,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,63,0.006154666965206464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,63,0.006218666831652324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,127,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,127,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,127,0.008767999708652496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,127,0.006565333033601443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,255,0.007920000081261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,255,0.011503999431928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,255,0.011642667154471079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,255,0.007354666789372762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,511,0.011413333316644033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,511,0.011770666887362799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,511,0.00725333330531915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,511,0.007983999947706858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,1023,0.016389333953460056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,1023,0.016821333517630894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,1023,0.007471999774376552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,1023,0.009152000149091085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,2047,0.01924266666173935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,2047,0.024501333634058636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,2047,0.007754666730761528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,2047,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,4095,0.024170666933059692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,4095,0.032442666590213776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,4095,0.007578666632374127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,4095,0.014117332796255747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,8191,0.0281333327293396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,8191,0.04491200049718221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,8191,0.007786666974425316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,8191,0.0229066660006841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,16383,0.03440000116825104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,16383,0.06927466889222463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,16383,0.008133333176374435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,16383,0.03186133255561193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,32767,0.03504000107447306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,32767,0.09237866600354512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,32767,0.007978666573762894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,32767,0.0528106689453125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,65535,0.03516799956560135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,65535,0.13969600200653076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,65535,0.008192000289758047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,65535,0.08513599634170532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,1,0.009749333063761393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,1,0.009674666449427605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,1,0.006031999985376994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,1,0.006261333202322324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,3,0.0100426667680343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,3,0.009717333440979322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,3,0.0061919999619325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,3,0.00595199999709924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,7,0.00978133330742518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,7,0.009759999811649323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,7,0.005994666367769241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,7,0.006266666576266289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,15,0.009829333052039146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,15,0.00966933307548364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,15,0.006277333324154218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,15,0.006026666611433029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,31,0.00973866693675518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,31,0.009663999701539675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,31,0.00600533311565717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,31,0.006293333445986112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,63,0.009701333319147428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,63,0.009621333330869675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,63,0.006234666953484218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,63,0.006026666611433029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,127,0.009679999823371569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,127,0.009701333319147428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,127,0.006346666564544042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,127,0.006666666517655055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,255,0.011855999628702799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,255,0.011610666910807291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,255,0.00720000018676122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,255,0.007541333635648091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,511,0.01626666635274887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,511,0.01672533278663953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,511,0.007301333049933116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,511,0.008538666491707167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,1023,0.021365332106749218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,1023,0.024234667420387268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,1023,0.007861333588759104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,1023,0.010239999741315842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,2047,0.027269333600997925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,2047,0.03586666782697042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,2047,0.0081386665503184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,2047,0.01628799984852473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,4095,0.03421866645415624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,4095,0.05269866685072581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,4095,0.008346666892369589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,4095,0.020346666375796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,8191,0.0341386670867602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,8191,0.06745600203673045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,8191,0.008346666892369589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,8191,0.029264000554879505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,16383,0.03426666557788849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,16383,0.08990933497746785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,16383,0.008101333553592363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,16383,0.04610133171081543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,32767,0.03430933256944021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,32767,0.13371200362841287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,32767,0.008218666538596153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,32767,0.07906133433183034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,65535,0.034287999073664345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,1,0.01020800011853377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,1,0.010149333626031876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,65535,0.22331732511520386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,65535,0.14214932918548584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,65535,0.008303999900817871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,1,0.006186666587988536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,1,0.006335999816656113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,3,0.010202666744589806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,3,0.010154666379094124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,3,0.006224000205596288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,15,0.010245333115259806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,3,0.006048000107208888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,7,0.010384000216921171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,7,0.010037333394090334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,7,0.006181333214044571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,7,0.006346666564544042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,15,0.010144000252087912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,15,0.006165333092212677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,15,0.006688000013430913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,31,0.010064000263810158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,31,0.010101333260536194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,31,0.006101333225766818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,63,0.006298666819930077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,31,0.006341333190600078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,63,0.010101333260536194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,63,0.0100853331387043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,63,0.005994666367769241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,127,0.010170666500926018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,127,0.010112000008424124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,255,0.007413333281874657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,127,0.006490666419267654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,127,0.006506666541099548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,255,0.01809599995613098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,255,0.017978666971127193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,255,0.007989333321650824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,511,0.02271999915440877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,511,0.02473066747188568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,511,0.00820266641676426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,2047,0.03540800015131632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,511,0.009674666449427605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,1023,0.03344533344109853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,1023,0.03977599988381068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,1023,0.008602666358153025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,4095,0.03572266548871994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,1023,0.016058667252461117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,2047,0.05077866713205973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,2047,0.008736000085870424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,2047,0.019839999576409657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,4095,0.06698666512966156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,4095,0.00850133349498113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,4095,0.02775999903678894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,8191,0.03566933423280716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,8191,0.08877866466840108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,8191,0.00847999999920527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,32767,0.03570666660865148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,8191,0.04504533112049103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,16383,0.035487999518712364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,16383,0.13370133439699808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,16383,0.0086666668454806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,16383,0.0771679977575938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,32767,0.2210986614227295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,32767,0.0086666668454806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,32767,0.14197867115338644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,65535,0.035445332527160645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,65535,0.39557866255442303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,1,0.007647999872763951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,65535,0.0086666668454806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,1,0.0075093333919843035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,65535,0.27188799778620404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,1,0.0058080001423756284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,1,0.005888000130653381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,3,0.007685333490371704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,3,0.007578666632374127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,7,0.007807999849319458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,3,0.005834666391213735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,15,0.007637333124876022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,3,0.005957333371043205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,7,0.007525333513816197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,7,0.005648000165820122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,7,0.00590933362642924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,15,0.007653333246707916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,15,0.005888000130653381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,15,0.00578666664659977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,31,0.007770666852593422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,31,0.00797333319981893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,31,0.005765333150823911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,31,0.005626666670044263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,63,0.008698666468262672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,63,0.008639999975760778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,63,0.005882666756709416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,63,0.005701333284378052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,127,0.008570666735370954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,127,0.00891733355820179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,127,0.006031999985376994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,127,0.0061493335912625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,255,0.008698666468262672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,255,0.008821333448092142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,255,0.00731733317176501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,255,0.007418666655818622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,511,0.008645333349704742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,511,0.009872000043590864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,511,0.006800000245372455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,511,0.007600000128149986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,1023,0.013663999736309052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,1023,0.015669333438078564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,2047,0.00679466687142849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,1023,0.007082666580875714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,1023,0.008303999900817871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,2047,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,4095,0.006757333253820737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,2047,0.016837333639462788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,2047,0.00901333304742972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,4095,0.012794667234023413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,4095,0.016607999801635742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,4095,0.010389333590865135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,8191,0.013141332815090815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,8191,0.014741333822409311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,8191,0.00684799998998642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,8191,0.012448000411192576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,16383,0.013221333424250284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,16383,0.01575999955336253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,16383,0.007093333328763644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,16383,0.014202666779359182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,32767,0.014954666296641031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,32767,0.018373332917690277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,32767,0.007109333450595538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,32767,0.017840000490347546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,65535,0.016517333686351776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,65535,0.02180800090233485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,65535,0.007034666836261749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,1,0.007696000238259633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,65535,0.022304000953833263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,1,0.005658666913708051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,131071,0.02958933264017105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,1,0.007765333478649457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,131071,0.020736000190178554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,131071,0.00707733320693175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,131071,0.02932800104220708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,1,0.0057386669019858045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,3,0.007610666876037915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,3,0.00772266648709774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,3,0.005941333249211311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,3,0.006389333556095759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,7,0.00749333327015241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,7,0.007978666573762894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,7,0.005632000043988228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,7,0.0059199997534354525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,15,0.007642666498819987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,15,0.007610666876037915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,15,0.006021333237489064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,15,0.006095999851822853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,31,0.007733333234985669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,31,0.00808533343176047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,31,0.005664000287652016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,31,0.005930666501323382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,63,0.0085333331177632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,63,0.0085333331177632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,63,0.00600533311565717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,63,0.006069333602984746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,127,0.008618666479984919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,127,0.008799999952316284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,127,0.005968000118931134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,127,0.006208000083764394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,511,0.008821333448092142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,255,0.008639999975760778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,255,0.008752000207702318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,255,0.007216000308593114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,255,0.0074346667776505155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,511,0.009743999689817429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,511,0.006757333253820737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,511,0.007594666754206021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,1023,0.013594667116800943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,1023,0.015562667200962702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,1023,0.0069386667261521024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,1023,0.008303999900817871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,4095,0.012666666259368261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,2047,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,2047,0.01674666628241539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,2047,0.006906666482488315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,2047,0.009136000027259191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,4095,0.012730666746695837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,4095,0.007178666690985362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,4095,0.010485333700974783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,8191,0.01301866645614306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,8191,0.014773332824309668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,8191,0.006927999978264173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,8191,0.011941333611806234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,16383,0.014149333039919535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,16383,0.016197333733240765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,16383,0.006725333631038666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,16383,0.014186666657527288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,32767,0.015125333021084467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,32767,0.01887999971707662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,32767,0.00701333334048589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,32767,0.017792000124851864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,65535,0.01711999997496605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,65535,0.022298666338125866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,65535,0.00690133310854435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,65535,0.023056000471115112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,131071,0.021231998999913532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,131071,0.03915733347336451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,1,0.007584000006318092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,1,0.007711999739209811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,1,0.005914666379491488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,131071,0.007231999809543292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,1,0.006128000095486641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,131071,0.03216533362865448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,3,0.007818666597207388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,3,0.007706666365265846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,3,0.0058133335163195925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,3,0.0058613332609335584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,7,0.007765333478649457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,7,0.007760000104705493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,7,0.006165333092212677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,7,0.005781333272655805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,15,0.007807999849319458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,15,0.007631999750932057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,15,0.005850666513045629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,63,0.008522666369875273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,15,0.006042666733264923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,31,0.008080000057816505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,31,0.008000000069538752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,31,0.006271999950210254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,31,0.005946666623155276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,63,0.008559999987483025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,63,0.005877333382765452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,63,0.006048000107208888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,127,0.008799999952316284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,127,0.008698666468262672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,127,0.006133333469430606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,127,0.006394666930039723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,255,0.008879999940594038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,255,0.008661333471536636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,255,0.00696000022192796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,255,0.007386666412154834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,511,0.008933333059151968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,511,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,511,0.007157333195209503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,511,0.007946666950980822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,1023,0.01379199946920077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,1023,0.015696000307798386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,1023,0.007061333085099856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,1023,0.008037333066264788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,2047,0.012266666938861212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,2047,0.012458667159080505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,2047,0.007050666958093643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,2047,0.009599999835093817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,4095,0.012698666503032049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,4095,0.012997332960367203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,4095,0.006927999978264173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,4095,0.010053333515922228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,8191,0.013295999417702356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,8191,0.015615999698638916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,8191,0.007104000076651573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,8191,0.012357333054145178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,16383,0.014015999933083853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,16383,0.01718933383623759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,16383,0.0069440001000960665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,16383,0.014159999787807465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,32767,0.017727999637524288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,32767,0.026346666117509205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,32767,0.006927999978264173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,32767,0.020213333268960316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,65535,0.01924266666173935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,65535,0.031983998914559685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,65535,0.007205333560705185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,65535,0.024453334510326385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,131071,0.02624000112215678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,1,0.007733333234985669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,131071,0.05507733424504598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,1,0.007600000128149986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,131071,0.007290666922926903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,1,0.005877333382765452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,3,0.00573333352804184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,131071,0.03734400123357773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,1,0.005626666670044263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,3,0.007791999727487564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,3,0.007733333234985669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,3,0.005679999788602193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,7,0.007840000092983246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,7,0.007834666719039282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,7,0.005728000154097875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,7,0.006042666733264923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,15,0.007690666864315669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,31,0.005797333394487699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,15,0.007834666719039282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,15,0.00578666664659977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,15,0.005685333162546158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,31,0.008047999814152718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,31,0.008053333188096682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,31,0.006181333214044571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,63,0.008650666723648706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,63,0.008592000231146812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,63,0.005824000264207522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,63,0.005893333504597346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,127,0.008799999952316284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,127,0.008933333059151968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,127,0.006480000292261441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,127,0.006709333509206772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,255,0.008714666590094566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,255,0.008842666943868002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,255,0.00696000022192796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,255,0.00749333327015241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,511,0.00874133345981439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,511,0.009743999689817429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,511,0.00706666645904382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,511,0.007989333321650824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,1023,0.013728000223636627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,1023,0.015664000064134598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,1023,0.007162666569153468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,1023,0.008389333263039589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,2047,0.013248000293970108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,2047,0.01664000004529953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,2047,0.007237333183487256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,2047,0.009381333366036415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,4095,0.012794667234023413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,4095,0.013349333157142004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,4095,0.0069919998447100324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,4095,0.01027199998497963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,8191,0.013552000125249227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,8191,0.015546667079130808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,8191,0.007157333195209503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,8191,0.012037333101034164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,16383,0.01444799949725469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,16383,0.016303999970356624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,16383,0.007045333584149678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,16383,0.014378666877746582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,32767,0.016117333124081295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,32767,0.020074666788180668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,32767,0.00690133310854435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,32767,0.0179626668492953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,65535,0.017530667285124462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,65535,0.02295999974012375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,65535,0.007231999809543292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,65535,0.022458667556444805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,1,0.007706666365265846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,1,0.007733333234985669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,1,0.00573333352804184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,131071,0.022101332743962605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,131071,0.039877332746982574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,1,0.0057386669019858045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,3,0.007626666376988093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,3,0.005989333614706993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,131071,0.031850665807724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,131071,0.007018666714429855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,3,0.007749333356817563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,3,0.005749333028992017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,15,0.007786666974425316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,7,0.007797333101431529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,7,0.007738666608929634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,7,0.00573333352804184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,7,0.006111999973654747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,15,0.0075519997626543045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,15,0.006026666611433029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,63,0.00878399983048439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,15,0.005749333028992017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,31,0.008000000069538752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,31,0.008069333309928576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,31,0.005935999875267346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,31,0.005685333162546158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,63,0.008496000121037165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,63,0.005930666501323382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,63,0.006128000095486641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,127,0.008858666444818178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,127,0.008890666688481966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,127,0.006266666576266289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,127,0.006208000083764394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,255,0.008767999708652496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,255,0.008736000085870424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,255,0.006981333096822103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,255,0.007605333502093951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,511,0.008879999940594038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,511,0.009808000177145004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,511,0.007141333073377609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,511,0.007711999739209811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,1023,0.01368533323208491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,1023,0.01573333392540614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,1023,0.007237333183487256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,1023,0.008240000034372011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,2047,0.012821332861979803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,2047,0.012629333883523941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,2047,0.007125333572427432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,2047,0.008896000062425932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,4095,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,4095,0.013343999783198038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,4095,0.007082666580875714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,4095,0.010159999753038088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,8191,0.013893333574136099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,8191,0.015893333901961643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,8191,0.0069759997228781385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,8191,0.012063999970753988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,16383,0.014453332871198654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,16383,0.017322666943073273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,16383,0.007167999943097432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,65535,0.020101333657900494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,16383,0.014538666854302088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,32767,0.017840000490347546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,32767,0.026922665536403656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,32767,0.006842666616042455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,32767,0.019695999721686046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,65535,0.03234666585922241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,65535,0.007327999919652939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,65535,0.02405333270629247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,131071,0.025968000292778015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,1,0.00785600021481514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,131071,0.05669866502285004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,1,0.007727999861041705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,1,0.005882666756709416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,131071,0.00707733320693175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,1,0.00578666664659977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,131071,0.035936000446478523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,3,0.007920000081261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,3,0.008058666562040647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,3,0.005744000275929769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,3,0.005759999776879947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,7,0.007765333478649457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,7,0.007658666620651881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,7,0.005925333127379417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,7,0.0058133335163195925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,15,0.00821333316465219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,15,0.007920000081261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,15,0.005717333406209946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,15,0.006031999985376994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,31,0.007903999959429106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,31,0.008037333066264788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,31,0.005935999875267346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,31,0.00625599982837836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,63,0.00855466661353906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,63,0.00902399979531765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,63,0.005775999898711841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,63,0.006021333237489064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,127,0.008826666822036108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,127,0.008858666444818178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,127,0.006144000217318535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,127,0.006448000048597653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,255,0.00878399983048439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,255,0.00878399983048439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,255,0.006911999856432279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,255,0.0074986666440963745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,511,0.00884799969693025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,511,0.009818666925032934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,511,0.0069333333522081375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,511,0.008058666562040647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,1023,0.011792000383138657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,1023,0.012047999848922094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,1023,0.00701333334048589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,1023,0.008223999912540117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,2047,0.012703999876976013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,2047,0.013232000172138214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,2047,0.0069919998447100324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,2047,0.009359999870260557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,4095,0.012965332716703415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,4095,0.013983999689420065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,4095,0.007061333085099856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,4095,0.01027199998497963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,8191,0.015909332782030106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,8191,0.020655999581019085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,8191,0.007007999966541926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,8191,0.013557333499193192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,16383,0.01700266698996226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,16383,0.024031999210516613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,16383,0.007055999711155891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,16383,0.01602666700879733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,32767,0.020506666352351505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,32767,0.03524799893299738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,32767,0.007178666690985362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,32767,0.02258133391539256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,65535,0.03310933212439219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,65535,0.024666666984558105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,65535,0.05428266525268555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,65535,0.007290666922926903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,131071,0.03223466624816259
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,1,0.005877333382765452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,131071,0.0890880028406779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,1,0.009914666414260864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,1,0.00973866693675518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,131071,0.007338666667540868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,1,0.005829333638151486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,131071,0.06157866617043813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,3,0.009952000031868616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,3,0.00984533317387104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,3,0.005925333127379417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,3,0.005999999741713206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,7,0.009743999689817429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,7,0.00996800015370051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,7,0.0058613332609335584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,7,0.006330666442712148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,15,0.009952000031868616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,15,0.009877333417534828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,15,0.00589866687854131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,15,0.006037333359320958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,31,0.00973866693675518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,31,0.009973333527644476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,31,0.005834666391213735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,31,0.0063573333124319715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,127,0.009770666559537252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,63,0.009973333527644476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,63,0.00978133330742518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,63,0.005999999741713206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,63,0.006464000170429547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,127,0.009962666779756546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,127,0.0063733334342638654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,255,0.007690666864315669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,127,0.006581333155433337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,255,0.011957333733638128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,255,0.012005332857370377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,255,0.007216000308593114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,511,0.016255999604860943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,511,0.017071999609470367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,511,0.007386666412154834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,511,0.008506666868925095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,1023,0.021935999393463135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,1023,0.024533333877722423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,1023,0.007850666840871176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,1023,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,2047,0.027461332579453785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,2047,0.03586666782697042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,2047,0.008181333541870117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,2047,0.016645333419243496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,4095,0.03465600063403448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,4095,0.0533493310213089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,4095,0.008170666793982187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,4095,0.021354667842388153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,8191,0.03437866767247518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,8191,0.06864533325036366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,8191,0.008341333518425623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,8191,0.03065066784620285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,16383,0.034373333056767784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,16383,0.0904319981733958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,16383,0.0468746672074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,16383,0.008496000121037165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,1,0.009925333162148794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,32767,0.03448000053564707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,1,0.010266666611035665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,32767,0.13592533270517984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,32767,0.008266666904091835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,32767,0.08028799792130788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,1,0.00600533311565717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,1,0.006330666442712148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,3,0.009888000165422758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,3,0.010575999816258749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,3,0.006154666965206464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,3,0.006058666855096817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,7,0.009872000043590864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,7,0.0100426667680343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,7,0.0061919999619325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,7,0.006282666698098183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,15,0.00985599992175897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,31,0.010079999764760336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,15,0.010197333370645842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,15,0.006069333602984746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,15,0.006170666466156642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,31,0.009984000275532404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,31,0.0063040001938740415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,31,0.006282666698098183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,63,0.009919999788204828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,63,0.010149333626031876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,63,0.006133333469430606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,63,0.006058666855096817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,127,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,127,0.0099093330403169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,127,0.006549333532651265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,127,0.006533333410819371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,255,0.017594666530688603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,255,0.01788266624013583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,255,0.007397333160042763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,255,0.008229333286484083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,511,0.022704000274340313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,511,0.02455466737349828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,511,0.00850133349498113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,511,0.009397333487868309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,1023,0.03241066634654999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,1023,0.039349332451820374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,2047,0.05213866631189982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,1023,0.008506666868925095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,1023,0.016176000237464905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,2047,0.03548266738653183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,2047,0.0086666668454806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,2047,0.019866666446129482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,4095,0.034847999612490334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,4095,0.06706133484840393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,4095,0.00879466657837232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,4095,0.027647999425729115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,8191,0.03522133330504099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,8191,0.09005332986513774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,8191,0.008629333227872849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,8191,0.04507199923197428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,16383,0.03501333296298981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,16383,0.13378133376439413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,16383,0.008650666723648706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,16383,0.07687999804814656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,1,0.01402666668097178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,32767,0.03513599932193756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,1,0.014325333138306936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,32767,0.22437334060668945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,32767,0.008896000062425932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,32767,0.14179199934005737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,1,0.006458666796485583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,1,0.006330666442712148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,3,0.01403733342885971
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,3,0.014058666924635569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,3,0.006335999816656113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,3,0.006448000048597653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,7,0.01440000037352244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,7,0.013893333574136099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,7,0.006394666930039723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,7,0.006239999706546466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,15,0.014165333161751429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,15,0.014106666048367819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,15,0.006453333422541618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,15,0.0063786668082078295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,31,0.0145066666106383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,31,0.01394133393963178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,31,0.0064853330453236895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,31,0.006239999706546466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,63,0.013983999689420065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,63,0.014085333794355392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,63,0.006394666930039723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,63,0.0063040001938740415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,127,0.0144213338692983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,127,0.013951999445756277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,127,0.0068800002336502075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,127,0.006917333230376244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,255,0.026416001220544178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,255,0.026288000245889027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,255,0.008272000278035799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,255,0.009455999980370203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,511,0.03587199995915095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,511,0.040021332601706185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,511,0.009733333562811216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,511,0.01544533297419548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,1023,0.03589333345492681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,1023,0.04980800052483877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,1023,0.009786666681369146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,1023,0.019648000597953796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,2047,0.03856533269087473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,2047,0.06693866848945618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,2047,0.009898666913310686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,2047,0.02752533306678136
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,4095,0.03828799972931544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,4095,0.08958933750788371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,4095,0.009733333562811216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,4095,0.04535466432571411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,8191,0.0384853333234787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,8191,0.13362133502960205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,8191,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,8191,0.0766186664501826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,16383,0.039146666725476585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,16383,0.22216532627741495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,16383,0.009818666925032934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,16383,0.1416853368282318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,32767,0.038634667793909706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,32767,0.39655999342600506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,32767,0.009786666681369146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,1,0.008005333443482717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,1,0.007962666451931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,32767,0.2674293319384257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,1,0.005626666670044263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,1,0.0075573331365982694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,3,0.007829333345095316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,3,0.007834666719039282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,3,0.0057920000205437345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,3,0.005744000275929769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,7,0.007941333577036858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,7,0.007936000203092894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,7,0.005621333296100299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,7,0.006181333214044571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,15,0.007770666852593422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,15,0.007914666707317034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,15,0.005695999910434087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,15,0.0058026667684316635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,31,0.008186666915814081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,31,0.0081386665503184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,31,0.005765333150823911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,31,0.006090666477878888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,63,0.008736000085870424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,63,0.00871999996403853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,63,0.005615999922156334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,63,0.005722666780153911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,127,0.008992000172535578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,127,0.009082666908701261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,127,0.006128000095486641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,127,0.006517333288987477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,255,0.008863999818762144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,255,0.008954666554927826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,255,0.00690133310854435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,255,0.007647999872763951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,511,0.00897066667675972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,511,0.010106666634480158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,511,0.0069866664707660675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,511,0.00814933329820633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,1023,0.013855999956528345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,1023,0.01569066693385442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,1023,0.0069973332186539965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,1023,0.008538666491707167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,2047,0.013605333864688873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,2047,0.013898666948080063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,2047,0.007050666958093643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,2047,0.00938666673998038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,4095,0.013914667069911957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,4095,0.014629332969586054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,4095,0.006906666482488315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,4095,0.010362666721145311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,8191,0.014650666465361914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,8191,0.01732800031701724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,8191,0.007045333584149678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,8191,0.012469333906968435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,16383,0.015360000232855478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,16383,0.01836799954374631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,16383,0.007045333584149678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,16383,0.014485333114862442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,32767,0.01874133323629697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,32767,0.02769600103298823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,32767,0.007055999711155891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,32767,0.020288000504175823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,65535,0.020917333662509918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,65535,0.0337119996547699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,65535,0.007034666836261749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,65535,0.025648000339667004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,131071,0.02717333287000656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,131071,0.0591786652803421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,131071,0.007296000296870868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,1,0.007807999849319458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,131071,0.03735466549793879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,1,0.007733333234985669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,1,0.005941333249211311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,1,0.005994666367769241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,3,0.007776000226537387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,3,0.008042666440208754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,3,0.005829333638151486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,3,0.006074666976928711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,15,0.008005333443482717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,7,0.007770666852593422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,7,0.007674666742483775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,7,0.005941333249211311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,7,0.005925333127379417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,15,0.007791999727487564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,15,0.005850666513045629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,63,0.00891733355820179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,15,0.006095999851822853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,31,0.008282666405042013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,31,0.00795199970404307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,31,0.005744000275929769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,31,0.00590933362642924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,63,0.008629333227872849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,63,0.00578666664659977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,63,0.006117333347598712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,127,0.00891733355820179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,127,0.009114666531483332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,127,0.006090666477878888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,127,0.00596266674498717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,255,0.009018666421373686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,255,0.008821333448092142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,255,0.00706666645904382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,255,0.0075626665105422335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,511,0.008832000195980072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,511,0.010191999996701876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,511,0.006911999856432279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,511,0.007365333537260692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,1023,0.011850666254758835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,1023,0.011589333415031433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,1023,0.00696000022192796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,1023,0.008298666526873907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,2047,0.013440000514189402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,2047,0.014346666634082794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,2047,0.006864000111818314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,2047,0.00897066667675972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,4095,0.014080000420411428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,8191,0.006949333474040031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,4095,0.01469333345691363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,4095,0.0069333333522081375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,4095,0.010399999717871347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,8191,0.01684800038735072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,8191,0.021498667697111767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,8191,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,16383,0.017808000246683758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,32767,0.02164799968401591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,16383,0.025045332809289295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,16383,0.007109333450595538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,65535,0.025413334369659424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,16383,0.015925332903862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,32767,0.036703998843828835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,32767,0.007135999699433644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,32767,0.022448000808556873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,65535,0.05693866809209188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,65535,0.007029333462317784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,65535,0.031914666295051575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,131071,0.032602667808532715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,1,0.007663999994595845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,131071,0.09240532914797465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,131071,0.00726400005320708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,1,0.008101333553592363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,131071,0.059749335050582886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,3,0.005877333382765452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,1,0.0058186668902635574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,1,0.005973333492875099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,3,0.007877333089709282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,3,0.008261333530147871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,3,0.005935999875267346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,7,0.008186666915814081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,7,0.00790933333337307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,7,0.0058080001423756284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,7,0.005712000032265981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,15,0.007962666451931
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,15,0.007834666719039282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,15,0.0058559998869895935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,31,0.005834666391213735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,15,0.005925333127379417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,31,0.008463999877373377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,31,0.00821333316465219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,31,0.005877333382765452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,63,0.008725333337982496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,63,0.008837333569924036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,63,0.0058080001423756284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,63,0.0058613332609335584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,127,0.00884799969693025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,127,0.00926399976015091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,127,0.006271999950210254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,127,0.006111999973654747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,255,0.008912000184257826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,255,0.009082666908701261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,255,0.0069866664707660675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,255,0.007402666533986728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,511,0.0069973332186539965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,511,0.012005332857370377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,511,0.011546666423479715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,511,0.007829333345095316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,1023,0.011605333536863327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,1023,0.012096000214417776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,1023,0.007061333085099856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,1023,0.008042666440208754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,2047,0.015791999797026317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,2047,0.018698666244745255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,2047,0.00726400005320708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,2047,0.009813333551088968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,4095,0.01670933390657107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,4095,0.020346666375796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,4095,0.007087999954819679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,4095,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,8191,0.019679999599854153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,16383,0.03871466716130575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,8191,0.02888533224662145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,8191,0.007360000163316727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,8191,0.015077333897352219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,16383,0.02309866746266683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,16383,0.00726400005320708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,16383,0.019648000597953796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,32767,0.028592000404993694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,32767,0.06187733511130015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,32767,0.007770666852593422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,32767,0.03383466601371765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,65535,0.0352906659245491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,65535,0.09537067015965779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,65535,0.00761600024998188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,65535,0.05183466772238413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,131071,0.03789866715669632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,1,0.010415999839703241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,1,0.010224000240365664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,131071,0.14573867122332254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,1,0.005989333614706993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,131071,0.008058666562040647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,1,0.006090666477878888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,131071,0.0906880001227061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,3,0.010512000570694605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,3,0.0103946669648091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,3,0.005941333249211311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,3,0.006133333469430606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,7,0.010442666709423065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,7,0.010591999938090643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,7,0.006117333347598712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,15,0.010543999572594961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,7,0.006026666611433029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,15,0.010314666976531347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,15,0.006026666611433029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,15,0.006122666721542676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,31,0.010469333579142889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,31,0.010389333590865135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,31,0.00613866684337457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,31,0.006234666953484218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,63,0.010368000095089277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,63,0.009984000275532404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,127,0.010362666721145311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,255,0.01814933369557063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,63,0.005978666866819064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,63,0.006037333359320958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,127,0.0103946669648091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,127,0.0063573333124319715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,127,0.00655466690659523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,255,0.017829333742459614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,255,0.00761600024998188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,255,0.008474666625261307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,511,0.022618666291236877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,511,0.024832000335057575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,511,0.008463999877373377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,511,0.009861333295702934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,1023,0.033045334120591484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,1023,0.03942399968703588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,1023,0.00874133345981439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,1023,0.01709866647919019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,2047,0.03541333228349686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,2047,0.05236800014972687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,2047,0.009098666409651438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,2047,0.020410666863123577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,4095,0.0354666660229365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,4095,0.06764799853165944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,4095,0.008634666601816813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,4095,0.028917332490285236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,8191,0.03532800078392029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,8191,0.08969066540400188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,8191,0.009008000294367472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,8191,0.0458133320013682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,16383,0.03527999917666117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,16383,0.13529599706331888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,1,0.014495999862750372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,16383,0.009237333511312803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,16383,0.07885866860548656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,1,0.014197333405415217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,1,0.006624000146985054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,1,0.006261333202322324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,3,0.014250667144854864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,3,0.01431999976436297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,3,0.006293333445986112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,3,0.006346666564544042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,7,0.014352000008026758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,7,0.01444799949725469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,7,0.006213333457708359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,7,0.006224000205596288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,15,0.0143306665122509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,15,0.01441066712141037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,15,0.006448000048597653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,15,0.006325333068768184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,31,0.014495999862750372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,31,0.01431999976436297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,31,0.006261333202322324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,31,0.006234666953484218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,63,0.013973332941532135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,63,0.0143306665122509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,63,0.0063680000603199005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,63,0.006261333202322324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,127,0.014261333892742792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,127,0.014159999787807465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,127,0.006704000135262807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,127,0.006757333253820737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,255,0.026186667382717133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,255,0.026543999711672466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,255,0.00847999999920527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,255,0.009423999736706415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,511,0.03633599976698557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,511,0.040031999349594116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,511,0.009989333028594652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,511,0.01553600033124288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,1023,0.035536001125971474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,1023,0.05091199775536855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,1023,0.010117333382368088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,1023,0.01947733387351036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,2047,0.0386559988061587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,2047,0.0674773355325063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,2047,0.00997866690158844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,2047,0.02794666588306427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,4095,0.03797333439191183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,4095,0.09115733702977498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,4095,0.010213333492477735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,4095,0.04504533112049103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,8191,0.038848000268141426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,8191,0.13395733634630838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,8191,0.009839999799927076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,8191,0.0772266685962677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,16383,0.03802666564782461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,16383,0.22524267435073853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,1,0.02293333411216736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,16383,0.0102186668664217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,1,0.02250666668017705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,1,0.006927999978264173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,1,0.00707733320693175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,16383,0.14126400152842203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,3,0.022304000953833263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,3,0.02271466702222824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,3,0.007386666412154834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,3,0.007167999943097432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,7,0.02236266682545344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,7,0.02237333357334137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,7,0.007109333450595538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,7,0.0069866664707660675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,15,0.022255999346574146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,15,0.022592000663280487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,15,0.007397333160042763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,15,0.007071999832987785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,31,0.022458667556444805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,31,0.007151999821265538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,31,0.022405333817005157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,31,0.007071999832987785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,63,0.022389332453409832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,63,0.022384000321229298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,63,0.007381333038210869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,63,0.0069919998447100324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,127,0.02219199885924657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,127,0.02258666604757309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,127,0.008112000301480293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,127,0.008047999814152718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,255,0.04307733476161957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,255,0.04314666489760081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,255,0.013663999736309052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,255,0.015322666615247726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,511,0.04287999868392944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,511,0.05225066840648651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,511,0.013370666652917862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,511,0.01937066639463107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,1023,0.043141335248947144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,1023,0.0662613312403361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,1023,0.013733333597580591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,1023,0.027664000789324444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,2047,0.045509333411852516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,2047,0.09202133615811665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,2047,0.01357866699496905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,2047,0.044954667488733925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,4095,0.013658666362365087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,4095,0.045968001087506614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,4095,0.13708266615867615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,4095,0.07723199824492137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,8191,0.046240001916885376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,8191,0.2256960074106852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,8191,0.013503999759753546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,8191,0.14171199997266135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,16383,0.04563199977080027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,16383,0.4025546709696452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,16383,0.0136266661187013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,1,0.014789332946141561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,16383,0.26946133375167847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,1,0.014565333724021912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,1,0.006511999915043513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,1,0.006133333469430606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,3,0.014570667097965876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,3,0.014720000326633453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,3,0.0063040001938740415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,3,0.0063680000603199005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,7,0.014666666587193808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,7,0.0144213338692983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,7,0.006389333556095759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,7,0.006106666599710782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,15,0.014495999862750372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,15,0.01461333284775416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,31,0.006560000280539195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,15,0.006346666564544042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,15,0.0063786668082078295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,31,0.014576000471909841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,31,0.014554666976133982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,31,0.006074666976928711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,63,0.014405333747466406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,63,0.014373333503802618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,63,0.006501333167155583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,63,0.006309333567818006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,127,0.014602666099866232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,127,0.014501333236694336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,127,0.0069386667261521024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,127,0.006768000001708667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,255,0.026373334228992462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,255,0.026506667335828144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,255,0.008592000231146812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,255,0.009632000078757605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,511,0.03622400015592575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,511,0.040218666195869446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,511,0.01028266673286756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,511,0.01587733378012975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,1023,0.035818666219711304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,1023,0.0497920016447703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,1023,0.010314666976531347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,1023,0.020256000260512035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,2047,0.03850133220354716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,2047,0.06840533514817555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,2047,0.010346666599313417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,2047,0.028762665887673695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,4095,0.03824000060558319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,4095,0.09106666843096416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,4095,0.010399999717871347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,4095,0.04605866471926371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,1,0.022757334013779957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,1,0.022789334257443745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,1,0.007034666836261749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,1,0.006949333474040031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,3,0.02250666668017705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,3,0.022624000906944275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,3,0.007151999821265538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,3,0.007061333085099856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,7,0.022826666633288067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,7,0.02279466638962428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,7,0.007104000076651573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,7,0.006906666482488315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,15,0.022570667167504627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,15,0.022639999787012737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,15,0.007231999809543292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,15,0.007018666714429855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,31,0.02271999915440877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,31,0.022629333039124806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,31,0.007061333085099856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,31,0.00696000022192796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,63,0.02242133269707362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,63,0.022485333184401195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,63,0.007269333427151044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,63,0.007045333584149678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,127,0.022517333428064983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,127,0.022389332453409832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,127,0.008000000069538752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,127,0.007930666829148928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,255,0.042634665966033936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,255,0.0429013321797053
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,255,0.013594667116800943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,255,0.015317333241303762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,511,0.04297600189844767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,511,0.05152533451716105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,511,0.013477332890033722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,511,0.01942933350801468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,1023,0.04260266820589701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,1023,0.027621333797772724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,1023,0.06643733382225037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,1023,0.013541333377361298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,2047,0.045552000403404236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,2047,0.0921013355255127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,2047,0.01350933313369751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,2047,0.04518933097521464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,4095,0.04549333453178406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,4095,0.13737066586812338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,4095,0.013530666629473368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,1,0.03875199953715006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,4095,0.07718400160471599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,1,0.03895466774702072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,1,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,1,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,3,0.03900266687075297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,3,0.03925866633653641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,3,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,3,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,7,0.0391839991013209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,7,0.038917332887649536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,7,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,7,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,15,0.03890133400758108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,15,0.03917866696914037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,15,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,15,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,31,0.039264000952243805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,31,0.039018665750821434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,31,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,31,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,63,0.03846933444341024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,63,0.038880000511805214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,63,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,63,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,127,0.03873066604137421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,127,0.038560000558694206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,127,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,127,0.012805332740147909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,255,0.04022400081157684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,255,0.04066666712363561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,255,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,255,0.016447999825080235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,511,0.041519999504089355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,511,0.05367999772230784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,511,0.012949333836634954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,511,0.025429333249727886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,1023,0.04093866546948751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,1023,0.0775786687930425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,1023,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,1023,0.0421013335386912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,2047,0.04507733384768168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,2047,0.12500799695650736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,2047,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,2047,0.07506666580835979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,4095,0.04468800127506256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,4095,0.216154674688975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,4095,0.012826666235923767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,4095,0.1377120018005371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,1,0.023333333432674408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,1,0.023306667804718018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,1,0.00744000015159448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,1,0.007125333572427432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,3,0.023178666830062866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,3,0.023530667026837666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,3,0.007189333438873291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,3,0.00701333334048589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,7,0.02342933416366577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,7,0.02332266668478648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,7,0.007344000041484833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,7,0.007093333328763644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,15,0.022874665757020313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,15,0.023237332701683044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,15,0.007125333572427432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,15,0.007034666836261749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,31,0.023381332556406658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,31,0.02346666653951009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,31,0.007280000175038974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,31,0.00706666645904382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,63,0.0227360005180041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,63,0.02310933421055476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,63,0.007093333328763644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,63,0.0069226666043202085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,127,0.023039999107519787
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,127,0.02313599983851115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,127,0.008341333518425623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,127,0.008234666660428047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,255,0.043322667479515076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,255,0.04347200194994608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,255,0.013882666826248169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,255,0.01575999955336253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,511,0.04349866509437561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,511,0.052671998739242554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,511,0.014058666924635569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,511,0.02035733312368393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,1023,0.043322667479515076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,1023,0.06706133484840393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,1023,0.013877333452304205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,1023,0.028917332490285236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,2047,0.045909335215886436
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,2047,0.09364266196886699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,2047,0.013946666071812311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,2047,0.046810666720072426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,1,0.03932799895604452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,1,0.03979733337958654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,1,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,1,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,3,0.03968533376852671
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,3,0.03982399900754293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,3,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,3,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,7,0.03989866624275843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,7,0.04081599911053976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,7,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,7,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,15,0.0399893323580424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,15,0.039877332746982574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,15,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,15,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,31,0.039706667264302574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,63,0.03937066594759623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,31,0.03963200002908707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,31,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,31,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,63,0.03961600114901861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,63,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,63,0.010650667051474253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,127,0.03939199944337209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,127,0.03923200070858002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,127,0.012768000364303589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,127,0.012757333616415659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,255,0.041322665909926094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,255,0.04125866790612539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,255,0.01302933320403099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,255,0.016496000190575916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,511,0.041509332756201424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,511,0.05499200026194254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,511,0.012784000486135483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,511,0.025397333006064098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,1023,0.04165866722663244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,1023,0.07845333218574524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,1023,0.013162666310866674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,1023,0.04223466912905375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,2047,0.045034666856129967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,2047,0.1253439982732137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,2047,0.012719999998807907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,1,0.07147199908892314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,2047,0.07444266478220622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,1,0.07167466481526692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,1,0.019920000185569126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,1,0.01825599993268649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,3,0.07163733243942261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,3,0.07135466734568278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,3,0.01844266677896182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,3,0.018661333868900936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,7,0.07141333321730296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,7,0.07180800040562947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,7,0.01883200059334437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,7,0.018346666047970455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,15,0.07169599831104279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,15,0.07124266525109609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,15,0.01844800015290578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,15,0.018730666488409042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,31,0.0710346649090449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,31,0.07168533404668172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,31,0.01887999971707662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,31,0.01841066653529803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,63,0.07070933282375336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,63,0.07060266534487407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,63,0.018415999909241993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,63,0.018592000007629395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,127,0.07065066695213318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,127,0.07098133365313213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,127,0.021877333521842957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,127,0.021370666722456615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,255,0.0775733341773351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,255,0.07723733286062877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,255,0.021541332205136616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,255,0.030320001145203907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,511,0.07733333110809326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,511,0.09769599636395772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,511,0.021695998807748158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,511,0.046853333711624146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,1023,0.07762133578459422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,1023,0.14245333274205527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,1023,0.02160000056028366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,1023,0.07871466875076294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,1,0.008069333309928576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,2047,0.08088533580303192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,1,0.007861333588759104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,2047,0.021770666042963665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,2047,0.14018133282661438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,2047,0.23465067148208618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,1,0.0057920000205437345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,1,0.006026666611433029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,3,0.0080960001796484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,3,0.007840000092983246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,3,0.005930666501323382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,3,0.006031999985376994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,7,0.007733333234985669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,7,0.007781333600481351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,7,0.005759999776879947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,7,0.006031999985376994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,15,0.007893333211541176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,15,0.007797333101431529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,15,0.00590933362642924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,15,0.006021333237489064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,31,0.007957333077987036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,31,0.008000000069538752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,31,0.005781333272655805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,31,0.0058026667684316635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,63,0.008687999720374743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,63,0.008858666444818178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,63,0.005914666379491488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,63,0.006053333481152852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,127,0.008816000074148178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,127,0.008912000184257826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,127,0.006335999816656113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,127,0.0063680000603199005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,255,0.00891733355820179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,255,0.009077333534757296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,255,0.007386666412154834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,255,0.007834666719039282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,511,0.008949333180983862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,511,0.009925333162148794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,511,0.007247999931375186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,511,0.00790933333337307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,1023,0.011861333002646765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,1023,0.011973333855470022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,1023,0.007290666922926903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,1023,0.008496000121037165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,2047,0.01379199946920077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,2047,0.014245333770910898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,2047,0.007280000175038974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,2047,0.009226666763424873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,4095,0.014266667266686758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,4095,0.015040000279744467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,4095,0.007338666667540868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,4095,0.010591999938090643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,16383,0.01830400029818217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,8191,0.0169813334941864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,8191,0.021989333132902782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,8191,0.007247999931375186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,8191,0.013621332744757334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,16383,0.025290665527184803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,16383,0.007418666655818622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,16383,0.016469333320856094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,32767,0.021781332790851593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,32767,0.03716266651948293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,32767,0.007290666922926903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,32767,0.022917332748572033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,65535,0.025744001070658367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,65535,0.007600000128149986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,65535,0.057018667459487915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,65535,0.03480533262093862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,1,0.008000000069538752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,1,0.007957333077987036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,1,0.00600533311565717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,1,0.006053333481152852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,3,0.007877333089709282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,3,0.008010666817426682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,131071,0.03469333300987879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,131071,0.09175999959309895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,131071,0.06252266466617584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,3,0.005914666379491488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,131071,0.007530666887760162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,3,0.006031999985376994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,7,0.007877333089709282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,7,0.008117333054542542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,7,0.0058666666348775225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,7,0.005770666524767876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,15,0.007925333455204964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,15,0.007776000226537387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,15,0.0060159998635451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,31,0.005685333162546158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,15,0.005872000008821487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,31,0.008117333054542542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,31,0.008357333640257517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,31,0.0057920000205437345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,63,0.00879466657837232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,63,0.008703999842206636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,63,0.006058666855096817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,63,0.005893333504597346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,127,0.008954666554927826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,127,0.008992000172535578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,127,0.006560000280539195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,127,0.006224000205596288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,255,0.009066666786869368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,255,0.00892800030608972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,255,0.006911999856432279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,255,0.007226666435599327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,511,0.01157333329319954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,511,0.011488000551859537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,1023,0.011786667009194693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,511,0.007045333584149678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,511,0.007514666765928268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,1023,0.011616000284751257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,1023,0.007087999954819679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,1023,0.008058666562040647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,2047,0.01617066686352094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,2047,0.018602666755517323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,2047,0.007007999966541926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,8191,0.028501334289709728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,2047,0.009962666779756546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,4095,0.016693333784739178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,4095,0.020581333587567013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,4095,0.007146666447321574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,4095,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,8191,0.019797333826621372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,8191,0.007376000285148621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,8191,0.016255999604860943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,16383,0.023013333479563396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,16383,0.038848000268141426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,16383,0.007530666887760162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,16383,0.01956266661485036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,32767,0.029338667790095013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,32767,0.06172800064086914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,32767,0.007567999884486198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,32767,0.03395200024048487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,65535,0.03475733349720637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,65535,0.09500799576441447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,65535,0.007594666754206021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,65535,0.05293866495291392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,1,0.007850666840871176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,1,0.007941333577036858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,1,0.0060159998635451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,131071,0.03730666637420654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,131071,0.14789866407712302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,131071,0.09076799949010213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,131071,0.007760000104705493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,1,0.005829333638151486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,3,0.007861333588759104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,3,0.008074666683872541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,3,0.006021333237489064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,3,0.005946666623155276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,7,0.008277333031098047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,7,0.008090666805704435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,7,0.006085333103934924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,7,0.006037333359320958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,15,0.007834666719039282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,15,0.00795199970404307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,15,0.005994666367769241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,15,0.00596266674498717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,31,0.008458666503429413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,31,0.008320000022649765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,31,0.006042666733264923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,31,0.006031999985376994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,63,0.00874133345981439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,63,0.008767999708652496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,63,0.005984000240763028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,63,0.006101333225766818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,127,0.00922133338948091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,127,0.009183999771873156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,127,0.006389333556095759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,127,0.006309333567818006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,255,0.011440000186363855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,255,0.011594666788975397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,255,0.00701333334048589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,255,0.007418666655818622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,511,0.011850666254758835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,511,0.011909333368142446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,511,0.007290666922926903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,511,0.007680000116427739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,1023,0.016261332978804905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,1023,0.01703466723362605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,1023,0.007071999832987785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,1023,0.008933333059151968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,2047,0.019365333020687103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,2047,0.023887999355793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,2047,0.007680000116427739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,2047,0.010608000059922537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,4095,0.02327466756105423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,4095,0.032111999889214836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,4095,0.007327999919652939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,4095,0.013701333353916803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,8191,0.027893332143624622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,8191,0.04577599962552389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,8191,0.016506666938463848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,8191,0.02223466585079829
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,16383,0.03342399994532267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,16383,0.06820799907048543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,16383,0.008037333066264788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,16383,0.030826665461063385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,32767,0.03499733408292135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,32767,0.007946666950980822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,32767,0.09276266892751057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,32767,0.050997331738471985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,65535,0.034346667428811394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,65535,0.1400213340918223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,65535,0.007983999947706858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,65535,0.08458133538564046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,131071,0.036559998989105225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,131071,0.2395253380139669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,float16,1,0.03999999910593033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,1,0.04004266609748205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,131071,0.007957333077987036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,131071,0.14665599664052328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,fp8,1,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,1,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,float16,3,0.04063999901215235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,3,0.0402399996916453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,fp8,3,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,3,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,float16,7,0.04045866678158442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,7,0.0402453343073527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,fp8,7,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,7,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,float16,15,0.040720000863075256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,15,0.04028266668319702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,fp8,15,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,15,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,float16,31,0.04035733391841253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,31,0.04009599983692169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,fp8,31,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,31,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,float16,63,0.03955200066169103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,63,0.03976533313592275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,fp8,63,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,63,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,float16,127,0.040005333721637726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,255,0.04225599765777588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,127,0.039664000272750854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,fp8,127,0.013082666943470636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,127,0.01321600005030632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,float16,255,0.04312000175317129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,fp8,255,0.013237333546082178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,255,0.017397332936525345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,float16,511,0.04293333490689596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,511,0.05545066793759664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,fp8,511,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,511,0.026464000344276428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,float16,1023,0.04271999994913737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,1023,0.07886399825414021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,fp8,1023,0.01320533330241839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,1023,0.043493335445721946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,float16,1,0.07302933434645335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,1,0.07200533151626587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,float16,7,0.07260266443093617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,fp8,1,0.018346666047970455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,1,0.01854933301607768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,float16,3,0.0727893312772115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,3,0.07246399919191997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,fp8,3,0.01874133323629697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,3,0.018624000251293182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,7,0.07204799850781758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,fp8,7,0.01858666663368543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,7,0.018394666413466137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,float16,15,0.07281066477298737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,15,0.0728000005086263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,fp8,15,0.018559999763965607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,15,0.01865600049495697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,float16,31,0.0729973316192627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,31,0.07274133463700612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,fp8,31,0.018624000251293182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,31,0.01838933303952217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,float16,63,0.07179200152556102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,63,0.07206400235493977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,fp8,63,0.01852799952030182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,63,0.018592000007629395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,float16,127,0.07313600182533264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,127,0.07189333438873291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,fp8,127,0.021802666286627453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,127,0.021365332106749218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,float16,255,0.07852800190448761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,255,0.07838933169841766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,fp8,255,0.02165866643190384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,255,0.03049066662788391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,fp8,511,0.021829334398110706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,511,0.0470719983180364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,float16,511,0.08017066617806752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,511,0.09762666622797649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,float16,1023,0.07764266431331635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,fp8,1023,0.021690666675567627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,1023,0.14382400115331015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,float16,1,0.13335999846458435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,1,0.1323466698328654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,1023,0.07896000146865845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,fp8,1,0.03269333392381668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,1,0.032357332607110344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,float16,3,0.13320533434549967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,3,0.13338133692741394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,fp8,3,0.03263466556866964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,3,0.03266133368015289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,float16,7,0.13221333424250284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,7,0.13312000036239624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,fp8,7,0.03316800047953924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,float16,15,0.13320533434549967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,15,0.1332373321056366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,fp8,15,0.03268799930810928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,15,0.03287466615438461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,float16,31,0.13251733779907227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,31,0.1332480013370514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,31,0.03252266595760981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,7,0.03267733256022135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,fp8,31,0.03277866790692011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,float16,63,0.1330933372179667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,63,0.1325653294722239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,fp8,63,0.03265066693226496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,63,0.032042667269706726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,float16,127,0.1346826652685801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,127,0.1350879967212677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,fp8,127,0.040762667854626976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,float16,255,0.1446506679058075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,127,0.040661332507928215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,255,0.1443839967250824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,fp8,255,0.04088533421357473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,255,0.05657599866390228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,float16,511,0.14457066853841147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,511,0.1830400029818217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,fp8,511,0.04050133377313614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,511,0.08781866232554118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,float16,1023,0.14544533689816794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,1023,0.2731093366940816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,fp8,1023,0.040261333187421165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,1023,0.14884266257286072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,float16,3,0.0735999991496404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,float16,1,0.07301333546638489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,1,0.07275733351707458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,fp8,1,0.019167999426523846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,1,0.018911999960740406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,3,0.07335466643174489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,fp8,3,0.018719999740521114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,3,0.018730666488409042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,float16,7,0.07338666419188182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,7,0.07302933434645335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,fp8,7,0.01884799947341283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,fp8,15,0.018543999642133713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,7,0.018911999960740406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,float16,15,0.07321066657702129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,float16,63,0.07271466652552287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,63,0.0724426656961441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,15,0.07321066657702129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,15,0.018853332847356796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,float16,31,0.07356800138950348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,31,0.07317866881688435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,fp8,31,0.018842666099468868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,31,0.01865600049495697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,fp8,63,0.018810667097568512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,63,0.01869333287080129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,float16,127,0.0735999991496404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,127,0.07407466570536296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,255,0.03216533362865448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,fp8,127,0.02252800017595291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,127,0.022437334060668945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,float16,255,0.08016533156236012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,255,0.0792906681696574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,fp8,255,0.022517333428064983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,float16,511,0.07965866724650066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,511,0.10038399696350098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,fp8,511,0.02248000105222066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,511,0.049029335379600525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,float16,1,0.13548800349235535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,1,0.13470932841300964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,fp8,1,0.032272001107533775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,1,0.03278933217128118
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,float16,3,0.1350933313369751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,7,0.0329066663980484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,3,0.13500266273816428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,fp8,3,0.03265066693226496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,3,0.032746667663256325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,float16,15,0.13501333196957907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,float16,7,0.13537599643071493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,7,0.13461866974830627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,fp8,7,0.03244800120592117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,15,0.03262399882078171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,15,0.13591466347376505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,fp8,15,0.0324799989660581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,float16,31,0.1357599993546804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,fp8,63,0.03263466556866964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,31,0.13615999619166055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,fp8,31,0.032655999064445496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,31,0.03258133431275686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,float16,63,0.13540266950925192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,63,0.13638400038083395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,63,0.03254399945338567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,float16,127,0.13805866241455078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,127,0.13872533043225607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,fp8,127,0.04115733255942663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,127,0.04132800052563349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,float16,255,0.1462559998035431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,255,0.14627200365066528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,fp8,255,0.042080000042915344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,255,0.057402665416399636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,float16,511,0.14738667011260986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,511,0.18712000052134195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,fp8,511,0.04172799984614054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,float16,1,0.2565386692682902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,511,0.08763733506202698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,1,0.2582346598307292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,fp8,1,0.06765866776307423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,1,0.0676693320274353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,float16,3,0.2590186595916748
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,fp8,3,0.06630933284759521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,3,0.2598293423652649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,float16,15,0.259333332379659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,3,0.06639466683069865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,float16,7,0.258240004380544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,7,0.2580053408940633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,fp8,7,0.06782933572928111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,7,0.06778666873772939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,fp8,15,0.06790400048096974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,15,0.258026659488678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,15,0.06717333197593689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,float16,31,0.2592373291651408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,31,0.2609813412030538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,fp8,31,0.06736533343791962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,31,0.06789333124955495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,float16,63,0.26401599248250324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,63,0.26523733139038086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,fp8,63,0.06637333333492279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,63,0.06624533236026764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,float16,127,0.26129599412282306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,127,0.26128532489140827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,fp8,127,0.07918933530648549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,127,0.07941866914431255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,float16,255,0.278874675432841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,255,0.2799626588821411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,fp8,255,0.07913599908351898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,255,0.10644267002741496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,fp8,511,0.0795306662718455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,1,0.008282666405042013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,511,0.16636266311009726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,1,0.007930666829148928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,1,0.006117333347598712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,1,0.005888000130653381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,3,0.008090666805704435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,3,0.008256000156203905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,511,0.3551093339920044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,float16,511,0.2811200022697449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,3,0.005999999741713206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,3,0.005973333492875099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,7,0.00790933333337307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,7,0.007903999959429106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,7,0.006159999718268712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,7,0.006271999950210254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,15,0.007936000203092894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,15,0.005941333249211311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,15,0.007877333089709282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,15,0.005968000118931134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,31,0.008154666672150293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,31,0.008165333420038223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,31,0.006159999718268712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,31,0.006341333190600078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,63,0.008805333326260248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,63,0.008901333436369896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,63,0.005935999875267346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,63,0.0063733334342638654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,127,0.00902399979531765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,127,0.009056000038981438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,127,0.006560000280539195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,127,0.006821333120266597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,255,0.008943999807039896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,255,0.009178666397929192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,255,0.007173333317041397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,255,0.007711999739209811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,511,0.011514666179815928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,511,0.011418666690587997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,511,0.007365333537260692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,511,0.008303999900817871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,1023,0.011503999431928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,1023,0.01179733375708262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,1023,0.007189333438873291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,1023,0.008352000266313553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,2047,0.016330666840076447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,2047,0.01863466699918111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,2047,0.00749333327015241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,2047,0.010234666367371878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,4095,0.016714667280515034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,4095,0.020597333709398907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,4095,0.007407999907930692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,4095,0.011498666057984034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,8191,0.020288000504175823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,8191,0.028373333315054577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,8191,0.007477333148320516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,8191,0.015552000453074774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,16383,0.023130667706330616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,16383,0.0394400010506312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,16383,0.007322666545708974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,16383,0.020821332931518555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,32767,0.02900800108909607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,32767,0.061941335598627724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,32767,0.007653333246707916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,32767,0.03518400092919668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,65535,0.03515200068553289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,65535,0.09510933359464009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,65535,0.05393599967161814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,65535,0.007786666974425316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,131071,0.038015998899936676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,1,0.007877333089709282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,131071,0.09636800487836202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,131071,0.14673067132631937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,131071,0.007920000081261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,1,0.0080960001796484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,1,0.005893333504597346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,1,0.005882666756709416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,3,0.008026666939258575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,3,0.007807999849319458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,3,0.006208000083764394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,3,0.006058666855096817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,7,0.008170666793982187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,7,0.005888000130653381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,7,0.008005333443482717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,7,0.0058080001423756284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,15,0.007887999837597212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,15,0.007776000226537387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,15,0.006330666442712148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,15,0.006064000229040782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,31,0.008341333518425623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,31,0.00816000004609426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,31,0.006159999718268712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,31,0.00590933362642924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,63,0.00874133345981439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,63,0.009535999968647957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,63,0.006079999729990959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,63,0.0060159998635451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,127,0.008933333059151968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,127,0.009103999783595404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,127,0.00620266670982043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,127,0.00600533311565717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,255,0.011989332735538483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,255,0.011519999553759893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,255,0.007135999699433644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,255,0.00744000015159448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,511,0.011391999820868174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,511,0.01166933278242747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,511,0.0069919998447100324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,511,0.007514666765928268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,1023,0.016389333953460056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,1023,0.017029333859682083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,1023,0.007178666690985362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,1023,0.008672000219424566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,2047,0.019109333554903667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,2047,0.024346667031447094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,2047,0.0074879998962084455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,2047,0.010634666929642359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,4095,0.023818666736284893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,4095,0.03260799994071325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,8191,0.007541333635648091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,16383,0.03421333432197571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,4095,0.007797333101431529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,4095,0.013440000514189402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,8191,0.027637332677841187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,8191,0.045034666856129967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,8191,0.02218666672706604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,16383,0.068271999557813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,16383,0.007829333345095316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,16383,0.030805334448814392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,32767,0.034874667723973594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,32767,0.09307199716567993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,32767,0.007989333321650824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,32767,0.051311999559402466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,65535,0.03451200077931086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,65535,0.00785600021481514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,65535,0.13961600263913473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,65535,0.08437333504358928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,1,0.009914666414260864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,1,0.009813333551088968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,1,0.006122666721542676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,1,0.006266666576266289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,131071,0.03637866675853729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,3,0.0099093330403169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,131071,0.2421226700146993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,131071,0.007930666829148928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,3,0.009786666681369146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,131071,0.14576533436775208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,3,0.006101333225766818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,3,0.00620266670982043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,7,0.009930666536092758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,7,0.0100853331387043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,7,0.006069333602984746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,7,0.006074666976928711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,15,0.009706666693091393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,15,0.010090666512648264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,15,0.006090666477878888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,15,0.006949333474040031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,31,0.009904000287254652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,31,0.010090666512648264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,31,0.006021333237489064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,31,0.00613866684337457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,63,0.009658666948477427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,63,0.009973333527644476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,63,0.005914666379491488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,63,0.006293333445986112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,127,0.009690666571259499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,127,0.009935999910036722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,127,0.0063573333124319715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,127,0.006480000292261441
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,255,0.012144000579913458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,255,0.012165332833925882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,255,0.007184000064929326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,255,0.007701333612203598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,511,0.01632533346613248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,511,0.01701333373785019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,511,0.007407999907930692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,511,0.008416000132759413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,1023,0.021738665799299877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,1023,0.0103946669648091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,2047,0.0164533331990242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,1023,0.02473066747188568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,1023,0.007647999872763951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,2047,0.02714666724205017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,2047,0.03568533311287562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,2047,0.008192000289758047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,4095,0.03425599883000056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,4095,0.05268266797065735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,4095,0.008320000022649765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,4095,0.02073066681623459
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,8191,0.03398400048414866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,8191,0.0663679987192154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,8191,0.00816000004609426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,8191,0.02890666574239731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,16383,0.03448000053564707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,16383,0.08920533458391826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,16383,0.008314666648705801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,16383,0.046096002062161766
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,32767,0.03430933256944021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,32767,0.13326932986577353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,32767,0.008336000144481659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,32767,0.07921066880226135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,65535,0.03453866640726725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,65535,0.2228426734606425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,65535,0.008261333530147871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,65535,0.14288000265757242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,1,0.008746666833758354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,1,0.0063680000603199005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,131071,0.03572800010442734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,131071,0.4015680154164632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,3,0.008687999720374743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,131071,0.008506666868925095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,3,0.006144000217318535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,7,0.008821333448092142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,7,0.0063573333124319715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,131071,0.27172799905141193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,15,0.009349333122372627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,15,0.006495999793211619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,31,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,31,0.006298666819930077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,63,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,63,0.006464000170429547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,127,0.011349332829316458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,255,0.012576000144084295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,127,0.007178666690985362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,255,0.007967999825874964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,511,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,511,0.008586666857202848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,1023,0.01913600042462349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,1023,0.010304000228643417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,2047,0.028570666909217834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,2047,0.012837332983811697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,4095,0.042026668787002563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,4095,0.019914666811625164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,8191,0.059877331058184304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,8191,0.02605333427588145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,16383,0.09191466371218364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,16383,0.045040001471837364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,32767,0.06799999872843425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,32767,0.13201066851615906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,1,0.010224000240365664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,1,0.006229333579540253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,65535,0.2228106657663981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,3,0.010378666842977205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,3,0.006069333602984746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,7,0.010319999729593595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,7,0.005935999875267346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,65535,0.11359467109044392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,15,0.010277333358923594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,15,0.006154666965206464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,31,0.010373333469033241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,31,0.006319999694824219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,63,0.010570666442314783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,63,0.006037333359320958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,127,0.010213333492477735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,127,0.00660800002515316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,255,0.01322666679819425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,255,0.008021333565314611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,511,0.019306667149066925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,511,0.009434666484594345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,1023,0.028101332485675812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,1023,0.014101333916187286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,2047,0.04321600000063578
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,2047,0.017551999539136887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,4095,0.0670826683441798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,4095,0.026517334083716076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,8191,0.04136000076929728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,8191,0.08724266290664673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,16383,0.12931733330090842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,16383,0.06461866696675618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,32767,0.21651200453440347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,32767,0.11129599809646606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,65535,0.3921866814295451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,1,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,65535,0.20814400911331177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,1,0.005872000008821487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,3,0.010645333677530289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,3,0.006106666599710782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,7,0.010703999549150467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,7,0.006229333579540253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,15,0.006293333445986112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,15,0.010362666721145311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,31,0.010474666953086853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,31,0.006159999718268712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,63,0.010565333068370819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,63,0.006106666599710782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,127,0.010602666685978571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,127,0.006511999915043513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,255,0.02130666623512904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,255,0.008165333420038223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,511,0.02824000020821889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,511,0.013045333325862885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,1023,0.046800002455711365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,1023,0.0195573332409064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,2047,0.06614399949709575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,2047,0.025589334468046825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,4095,0.08480000495910645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,4095,0.039605334401130676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,8191,0.12837333480517069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,8191,0.06372266511122386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,16383,0.2143146594365438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,16383,0.11015466849009196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,32767,0.39050134023030597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,1,0.008613333106040955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,32767,0.20517865816752115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,1,0.007178666690985362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,65535,0.3944106499354045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,65535,0.7418826421101888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,3,0.008645333349704742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,3,0.005994666367769241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,7,0.008570666735370954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,7,0.006197333335876465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,15,0.008746666833758354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,15,0.006085333103934924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,31,0.009418666362762451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,31,0.005946666623155276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,63,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,127,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,63,0.006010666489601135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,127,0.006469333544373512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,255,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,255,0.007914666707317034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,511,0.01533866673707962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,511,0.008063999935984612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,1023,0.016143999993801117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,1023,0.009893333539366722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,2047,0.01640533283352852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,2047,0.009568000212311745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,4095,0.01651200031240781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,4095,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,8191,0.017680000513792038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,8191,0.013082666943470636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,16383,0.019744000087181728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,16383,0.015173333386580149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,32767,0.023557332654794056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,32767,0.018906666586796444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,1,0.008383999889095625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,1,0.006090666477878888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,65535,0.02808533360560735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,65535,0.02346666653951009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,131071,0.03894400099913279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,131071,0.030618667602539062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,3,0.00850133349498113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,3,0.006111999973654747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,7,0.00847999999920527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,7,0.006757333253820737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,15,0.00874133345981439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,15,0.0058453331391016645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,31,0.009482666850090027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,31,0.005658666913708051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,63,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,63,0.005754666402935982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,127,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,127,0.006517333288987477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,255,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,255,0.0074346667776505155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,511,0.01540800059835116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,511,0.007797333101431529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,1023,0.01639466608564059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,1023,0.008474666625261307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,2047,0.01634666696190834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,2047,0.009429333110650381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,4095,0.014896000425020853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,4095,0.010597333312034607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,8191,0.018474667022625606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,8191,0.012319999436537424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,16383,0.01995733380317688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,16383,0.014757333944241205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,32767,0.024293333292007446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,32767,0.01878400022784869
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,65535,0.02991466720898946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,1,0.008634666601816813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,1,0.005850666513045629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,3,0.00854399986565113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,3,0.005957333371043205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,7,0.008629333227872849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,65535,0.023599999646345775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,131071,0.05899199843406677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,7,0.005850666513045629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,131071,0.03375466664632162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,15,0.008997333546479544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,15,0.0058186668902635574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,31,0.009653333574533463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,31,0.00578666664659977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,63,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,63,0.005781333272655805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,127,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,127,0.006128000095486641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,255,0.01139733319481214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,255,0.007967999825874964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,511,0.01543466622630755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,511,0.007738666608929634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,1023,0.01635733370979627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,1023,0.008272000278035799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,2047,0.014688000082969666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,2047,0.009392000113924345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,4095,0.015498666713635126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,4095,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,8191,0.018901333212852478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,8191,0.012517333030700684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,16383,0.021189334491888683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,16383,0.014975999792416891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,32767,0.035562666753927864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,32767,0.020954666038354237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,1,0.008538666491707167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,65535,0.04906666775544485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,1,0.006133333469430606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,65535,0.02681066592534383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,3,0.00877333308259646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,3,0.006186666587988536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,131071,0.08277866741021474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,131071,0.04188799858093262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,7,0.008576000109314919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,7,0.00624533308049043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,15,0.008922666932145754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,15,0.00625599982837836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,31,0.009653333574533463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,31,0.006154666965206464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,63,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,63,0.006026666611433029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,127,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,127,0.006575999781489372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,255,0.011349332829316458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,255,0.008250666782259941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,511,0.015669333438078564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,511,0.008266666904091835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,1023,0.016751999656359356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,1023,0.00867733359336853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,2047,0.016469333320856094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,2047,0.00956266683836778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,4095,0.015546667079130808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,4095,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,8191,0.019146667172511418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,8191,0.012821332861979803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,16383,0.02056533346573512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,16383,0.015114666273196539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,32767,0.02548266698916753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,32767,0.01889066646496455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,65535,0.03242133309443792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,65535,0.02401600033044815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,1,0.008698666468262672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,131071,0.06132266422112783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,1,0.005728000154097875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,3,0.00877333308259646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,131071,0.03475733349720637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,3,0.0058666666348775225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,7,0.008752000207702318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,7,0.006064000229040782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,15,0.008949333180983862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,15,0.005877333382765452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,31,0.00955200009047985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,31,0.0058666666348775225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,63,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,63,0.005893333504597346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,127,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,127,0.006650666395823161
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,255,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,255,0.007701333612203598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,511,0.015429332852363586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,511,0.007925333455204964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,1023,0.01666133354107539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,1023,0.008341333518425623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,2047,0.015050667027632395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,2047,0.009589333087205887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,4095,0.016042667130629223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,4095,0.010570666442314783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,8191,0.019567999988794327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,8191,0.01258133351802826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,16383,0.021882665654023487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,16383,0.014890667051076889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,32767,0.035232000052928925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,32767,0.021231998999913532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,65535,0.05019199848175049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,65535,0.026719999810059864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,131071,0.0832533339659373
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,1,0.008890666688481966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,1,0.005717333406209946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,3,0.008672000219424566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,131071,0.040752001106739044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,3,0.006090666477878888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,7,0.008703999842206636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,7,0.0058026667684316635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,15,0.008997333546479544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,15,0.006037333359320958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,31,0.009365333244204521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,31,0.00573333352804184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,63,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,63,0.00620266670982043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,127,0.01139733319481214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,127,0.006170666466156642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,255,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,255,0.007711999739209811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,511,0.015263999501864115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,511,0.007791999727487564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,1023,0.013712000101804733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,1023,0.008309333274761835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,2047,0.01533866673707962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,2047,0.009269333134094873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,4095,0.01661866654952367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,8191,0.014111999422311783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,4095,0.010458666831254959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,8191,0.025914666553338368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,65535,0.04979733129342397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,16383,0.031194667021433514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,16383,0.017130666722853977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,32767,0.0517493337392807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,32767,0.024698667228221893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,65535,0.08027733365694682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,131071,0.13705066839853922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,1,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,1,0.006144000217318535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,131071,0.06764799853165944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,3,0.010687999427318573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,3,0.006330666442712148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,7,0.010533332824707031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,7,0.006405333057045937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,15,0.010346666599313417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,15,0.006394666930039723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,31,0.0103946669648091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,31,0.006250666454434395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,63,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,63,0.006277333324154218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,127,0.010474666953086853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,127,0.006762666627764702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,255,0.008576000109314919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,255,0.013104000439246496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,511,0.00960533320903778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,511,0.01937599976857503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,1023,0.02842666705449422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,1023,0.015647999942302704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,2047,0.04428266485532125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,2047,0.01893866683046023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,4095,0.06837333242098491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,4095,0.029466666281223297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,8191,0.08929066856702168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,8191,0.043434664607048035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,16383,0.12995733817418417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,1,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,1,0.005930666501323382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,16383,0.06810666620731354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,3,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,3,0.005994666367769241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,32767,0.2196320096651713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,15,0.010656000425418219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,32767,0.11512533823649089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,7,0.006213333457708359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,7,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,15,0.006090666477878888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,31,0.010640000303586325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,31,0.006064000229040782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,63,0.010709332923094431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,63,0.0058399997651577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,127,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,127,0.006826666494210561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,255,0.021253332495689392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,255,0.008874666566650072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,511,0.0281333327293396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,511,0.014080000420411428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,1023,0.04618666569391886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,1023,0.019882666567961376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,2047,0.06659733255704244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,2047,0.025839999318122864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,4095,0.08678932984670003
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,4095,0.04029333343108495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,8191,0.12795199950536093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,8191,0.06482666730880737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,1,0.01575999955336253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,16383,0.11181867122650146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,1,0.006613333399097125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,16383,0.21742399533589682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,3,0.015216000378131866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,32767,0.3961973190307617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,32767,0.2050666610399882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,3,0.006682666639486949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,7,0.015626666446526844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,7,0.006560000280539195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,15,0.015381333728631338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,15,0.006506666541099548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,31,0.015311999867359797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,31,0.0068800002336502075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,63,0.015253332753976187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,63,0.006453333422541618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,127,0.015578666081031164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,127,0.007290666922926903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,255,0.03019733230272929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,255,0.013023999830087027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,511,0.046949331959088646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,511,0.019173332800467808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,1023,0.06285333136717479
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,1023,0.024879999458789825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,2047,0.08549867073694865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,2047,0.03976533313592275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,4095,0.12826133767763773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,4095,0.0639626681804657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,8191,0.21860265731811523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,8191,0.11127466956774394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,16383,0.20508267482121786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,16383,0.3938933213551839
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,1,0.008725333337982496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,1,0.006581333155433337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,3,0.008736000085870424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,32767,0.39555732409159344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,32767,0.7476692994435629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,3,0.0063146669417619705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,7,0.008965333302815756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,7,0.005957333371043205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,15,0.009039999917149544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,15,0.006208000083764394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,63,0.011424000064531961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,31,0.00956266683836778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,255,0.011567999919255575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,31,0.006181333214044571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,63,0.006415999804933866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,127,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,127,0.006634666894872983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,255,0.0080960001796484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,511,0.015674666812022526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,511,0.008133333176374435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,1023,0.016778666526079178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,1023,0.008757333581646284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,2047,0.016234666109085083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,2047,0.00966933307548364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,4095,0.016858667135238647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,4095,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,8191,0.020608000457286835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,8191,0.012938667088747025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,16383,0.023290666441122692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,16383,0.015381333728631338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,32767,0.03704000016053518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,32767,0.021850667893886566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,65535,0.052426666021347046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,65535,0.028357334434986115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,131071,0.08492799599965413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,1,0.008912000184257826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,1,0.0058186668902635574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,3,0.008863999818762144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,131071,0.04258666435877482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,3,0.005893333504597346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,7,0.00891733355820179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,7,0.006095999851822853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,15,0.009061333412925402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,15,0.005888000130653381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,31,0.009674666449427605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,31,0.006010666489601135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,63,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,63,0.00589866687854131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,127,0.011477333803971609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,127,0.006442666674653689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,255,0.011567999919255575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,255,0.008000000069538752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,511,0.015557333827018738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,511,0.008005333443482717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,1023,0.013445333888133367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,1023,0.008277333031098047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,2047,0.01659199967980385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,2047,0.009535999968647957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,4095,0.017658667018016178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,4095,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,8191,0.027093333502610523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,8191,0.01394133393963178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,16383,0.032799998919169106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,16383,0.016986666868130367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,32767,0.054602667689323425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,32767,0.02475200096766154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,1,0.0057386669019858045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,65535,0.08161066472530365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,1,0.008949333180983862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,65535,0.04960533479849497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,3,0.00895999992887179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,3,0.006031999985376994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,7,0.008687999720374743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,131071,0.13777599732081094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,7,0.006122666721542676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,131071,0.06759466727574666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,15,0.009008000294367472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,15,0.005999999741713206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,31,0.009818666925032934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,31,0.006058666855096817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,63,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,127,0.011525332927703857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,63,0.0058613332609335584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,127,0.006384000182151794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,255,0.011514666179815928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,255,0.007920000081261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,2047,0.021354667842388153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,511,0.013050666699806849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,4095,0.024826665719350178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,511,0.008026666939258575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,1023,0.013258667041858038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,1023,0.008656000097592672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,2047,0.01033599985142549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,4095,0.011909333368142446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,8191,0.036271999279658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,8191,0.01586666703224182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,16383,0.0554720014333725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,16383,0.027056001126766205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,32767,0.08378133177757263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,32767,0.04048000027736028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,65535,0.13702399532000223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,65535,0.07073066631952922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,1,0.010741333166758219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,1,0.007082666580875714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,3,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,3,0.006394666930039723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,131071,0.24103999137878418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,131071,0.1153706709543864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,7,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,7,0.00620266670982043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,15,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,15,0.006442666674653689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,31,0.010741333166758219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,31,0.006266666576266289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,63,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,63,0.006528000036875407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,127,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,127,0.0075519997626543045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,255,0.021488000949223835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,255,0.009397333487868309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,511,0.028560000161329906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,511,0.015285332997639975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,1023,0.04673066735267639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,1023,0.022181332111358643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,2047,0.0664213349421819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,2047,0.028255999088287354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,4095,0.08740267157554626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,4095,0.04267199834187826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,8191,0.13102400302886963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,8191,0.06677866478761037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,1,0.015189333508412043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,1,0.006709333509206772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,16383,0.22107734282811484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,3,0.016042667130629223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,16383,0.11620799700419109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,3,0.006597333277265231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,7,0.015530666957298914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,7,0.006469333544373512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,15,0.015295999745527903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,15,0.006837333242098491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,31,0.01533866673707962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,31,0.006837333242098491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,63,0.01540800059835116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,63,0.006618666773041089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,127,0.015471999843915304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,127,0.007311999797821045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,255,0.030565333863099415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,255,0.013642666240533194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,511,0.047322665651639305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,511,0.019744000087181728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,1023,0.061861331264177956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,1023,0.02607999990383784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,2047,0.0862506628036499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,2047,0.04027199993530909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,4095,0.12898666659990946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,4095,0.06440000236034393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,1,0.025125332176685333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,8191,0.22105600436528525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,8191,0.11239467064539592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,1,0.009722666814923286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,3,0.025055999557177227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,3,0.009712000067035357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,7,0.02498133232196172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,16383,0.2073919971783956
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,7,0.009712000067035357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,15,0.0252960001428922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,16383,0.4007893403371175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,15,0.009663999701539675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,31,0.02500266581773758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,31,0.009733333562811216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,63,0.025360000630219776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,63,0.009626666704813639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,127,0.02508266766866048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,127,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,255,0.0499893327554067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,255,0.01905599981546402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,511,0.06513600051403046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,511,0.025146665672461193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,1023,0.08425066868464152
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,1023,0.0402399996916453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,2047,0.13155733545621237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,2047,0.06410133341948192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,4095,0.11214400331179301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,4095,0.2230293353398641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,1,0.01602666700879733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,8191,0.20683733622233072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,3,0.01590399940808614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,1,0.007114666824539502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,8191,0.40376532077789307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,3,0.00684799998998642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,7,0.016117333124081295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,16383,0.7637759844462076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,16383,0.39689600467681885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,7,0.00696000022192796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,15,0.01623999948302905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,15,0.007045333584149678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,31,0.015610666324694952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,255,0.031013332307338715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,31,0.007071999832987785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,63,0.0069226666043202085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,63,0.01562133307258288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,127,0.00821333316465219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,127,0.015775999675194424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,255,0.015423999478419622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,511,0.0476800004641215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,511,0.021850667893886566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,1023,0.0625600020090739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,1023,0.028677334388097126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,2047,0.08686400453249614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,2047,0.04286933441956838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,4095,0.1306880017121633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,1,0.02513599892457326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,1,0.009765333185593287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,4095,0.06659733255704244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,3,0.009674666449427605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,3,0.025616000096003216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,7,0.025370667378107708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,7,0.009893333539366722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,15,0.025360000630219776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,15,0.00979200005531311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,31,0.025205334027608235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,31,0.009754666437705358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,63,0.02498133232196172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,63,0.009904000287254652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,127,0.025434667865435284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,127,0.011503999431928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,255,0.05000533163547516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,255,0.019925333559513092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,511,0.0650186687707901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,511,0.027034667630990345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,1023,0.08402666449546814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,1023,0.04078399886687597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,2047,0.13211733102798462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,2047,0.06470400094985962
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,1,0.043621331453323364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,1,0.013749333719412485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,3,0.043978666265805565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,3,0.014010666559139887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,4095,0.2258560061454773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,4095,0.11225600043932597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,7,0.044394666949907936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,7,0.013861333330472311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,127,0.045141334335009255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,15,0.04389866689840952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,15,0.013733333597580591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,31,0.043968002001444496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,31,0.01394133393963178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,63,0.043893332282702126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,63,0.013701333353916803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,127,0.015824000040690105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,255,0.053743998209635414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,255,0.021402666966120403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,511,0.07392000158627827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,511,0.037248000502586365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,1023,0.11771733562151591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,1023,0.060864001512527466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,4095,0.20242132743199667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,2047,0.2145706613858541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,2047,0.10823466380437215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,4095,0.3980533281962077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,1,0.025909334421157837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,1,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,3,0.026517334083716076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,3,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,7,0.025914666553338368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,7,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,15,0.0259253333012263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,15,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,31,0.02589333305756251
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,31,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,63,0.02619733413060506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,63,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,127,0.026074667771657307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,127,0.013045333325862885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,255,0.051370665431022644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,255,0.02180800090233485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,1023,0.08558400472005208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,511,0.06596800188223521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,511,0.02932800104220708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,1023,0.04369066655635834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,2047,0.1340053379535675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,1,0.04426133135954539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,1,0.013733333597580591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,2047,0.06774400174617767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,3,0.04515199859937032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,3,0.013728000223636627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,7,0.044346665342648826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,7,0.013893333574136099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,15,0.013839999834696451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,15,0.044394666949907936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,31,0.044453332821528115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,31,0.013823999712864557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,255,0.05493866900602976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,63,0.04470933477083842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,63,0.01370666672786077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,127,0.04538666705290476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,127,0.01646399994691213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,255,0.0229066660006841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,511,0.07392000158627827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,511,0.037818667789300285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,1023,0.11943466464678447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,1023,0.06123200058937073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,2047,0.21674132347106934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,1,0.08027733365694682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,1,0.023520000278949738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,2047,0.10839466253916423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,3,0.08019199967384338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,3,0.023658665517965954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,7,0.08046400050322215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,7,0.02359466751416524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,15,0.08029866715272267
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,15,0.023658665517965954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,31,0.08079466720422109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,31,0.023621333142121632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,63,0.02364266663789749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,63,0.08049599826335907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,127,0.08253333469231923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,127,0.028463999430338543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,255,0.09591466188430786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,255,0.04191466669241587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,511,0.13194666306177774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,511,0.06486933430035909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,1023,0.2146079937616984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,1023,0.11009066303571065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,1,0.008938666433095932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,2047,0.20375466346740723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,2047,0.403221329053243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,1,0.006229333579540253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,3,0.008853333070874214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,3,0.006106666599710782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,7,0.00901333304742972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,7,0.006341333190600078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,15,0.009136000027259191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,15,0.006042666733264923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,31,0.009663999701539675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,31,0.00595199999709924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,63,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,63,0.006266666576266289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,127,0.011429333438475927
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,127,0.006688000013430913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,255,0.011584000041087469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,255,0.00797333319981893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,511,0.01587733378012975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,511,0.008223999912540117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,1023,0.013274667163689932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,1023,0.008714666590094566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,2047,0.01632533346613248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,8191,0.014725333700577417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,2047,0.009813333551088968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,4095,0.01834133391578992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,4095,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,8191,0.027098665634791057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,16383,0.03414933383464813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,16383,0.018063999712467194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,32767,0.05379199981689453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,32767,0.025013332565625507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,1,0.00890666681031386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,1,0.005829333638151486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,3,0.00891733355820179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,65535,0.08200533191363017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,65535,0.05193066596984863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,3,0.005744000275929769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,7,0.008799999952316284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,7,0.005834666391213735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,15,0.009173333023985228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,15,0.006117333347598712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,63,0.005984000240763028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,31,0.009749333063761393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,31,0.005770666524767876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,131071,0.06996266543865204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,63,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,131071,0.13796266913414001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,127,0.011434666812419891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,127,0.006362666686375936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,255,0.011461333682139715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,255,0.008042666440208754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,511,0.012618667135636011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,1023,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,511,0.008010666817426682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,1023,0.00850133349498113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,2047,0.021482666333516438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,2047,0.010341333225369453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,4095,0.024336000283559162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,4095,0.012069333344697952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,8191,0.03611200054486593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,8191,0.016234666109085083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,16383,0.05496533215045929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,16383,0.028058665494124096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,32767,0.04054400076468786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,32767,0.08363200227419536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,65535,0.13619200388590494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,1,0.008976000050703684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,65535,0.06996800005435944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,1,0.0058399997651577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,3,0.008943999807039896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,3,0.005893333504597346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,7,0.008912000184257826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,15,0.009119999905427298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,7,0.005994666367769241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,15,0.006229333579540253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,31,0.009626666704813639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,31,0.006111999973654747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,63,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,63,0.005759999776879947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,127,0.011648000528415045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,511,0.01333333303531011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,127,0.006239999706546466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,131071,0.11522133151690166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,255,0.012826666235923767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,131071,0.23871999979019165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,255,0.007685333490371704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,511,0.008277333031098047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,1023,0.01939733326435089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,1023,0.009322666873534521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,2047,0.028165332973003387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,2047,0.011653333902359009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,4095,0.04078399886687597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,4095,0.018266666680574417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,8191,0.05914666752020518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,8191,0.024458666642506916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,16383,0.0900320013364156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,16383,0.042709335684776306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,32767,0.13169599572817484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,32767,0.0652106652657191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,65535,0.22549867630004883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,65535,0.11062399546305339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,1,0.04548266530036926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,1,0.015178666760524115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,3,0.045738667249679565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,3,0.015263999501864115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,7,0.045696000258127846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,7,0.01552533358335495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,131071,0.4197546641031901
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,131071,0.2021440068880717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,15,0.045370668172836304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,15,0.01524266724785169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,31,0.04572799801826477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,31,0.01522133375207583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,63,0.04515199859937032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,63,0.015578666081031164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,127,0.04663999875386556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,127,0.019007999449968338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,255,0.05615466833114624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,255,0.026149332523345947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,511,0.07660800218582153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,511,0.041082667807737984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,3,0.08084799846013387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,1,0.08090133468310039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,1023,0.12215999762217204
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,1023,0.06465066472689311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,1,0.023669332265853882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,3,0.02382933348417282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,7,0.08161066472530365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,7,0.023792001108328503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,15,0.08078933258851369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,15,0.023872000475724537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,31,0.08057599763075511
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,31,0.024031999210516613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,63,0.08076799909273784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,63,0.023605334262053173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,127,0.08388266960779826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,127,0.031045332551002502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,255,0.09713066617647807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,255,0.042778665820757546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,511,0.13269866506258646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,511,0.06629866858323415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,1,0.15108799934387207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,1,0.04632533093293508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,1023,0.21805334091186523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,3,0.14949867129325867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,1023,0.11174399654070537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,3,0.04645866652329763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,7,0.1495786706606547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,7,0.04580266773700714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,15,0.1509706676006317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,15,0.04670399924119314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,31,0.15308266878128052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,31,0.0455626646677653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,63,0.15319466590881348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,63,0.046725332736968994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,255,0.17997866868972778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,127,0.054832001527150474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,127,0.15242133537928262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,255,0.07422400017579396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,511,0.24781332413355509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,511,0.11467199524243672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,1023,0.4055360158284505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,1023,0.1989120046297709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,1,0.08333333333333333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,1,0.026848000784715016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,3,0.08262933293978374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,3,0.0269813338915507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,7,0.08333866794904073
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,7,0.0266239990790685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,15,0.08331733445326488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,15,0.026320000489552815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,31,0.082997332016627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,31,0.026122666895389557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,63,0.08448533217112224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,63,0.02643733223279317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,127,0.08570667107899983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,127,0.03805333375930786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,255,0.10031466682751973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,255,0.049072002371152244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,511,0.13661332925160727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,1,0.1513866682847341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,511,0.07077333331108093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,1,0.04822933177153269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,3,0.15153066317240396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,3,0.04924799998601278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,7,0.15094400445620218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,7,0.04877866804599762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,15,0.15173332889874777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,15,0.04817600051561991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,31,0.15454933047294617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,31,0.04916266600290934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,63,0.15441067020098367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,63,0.049216002225875854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,127,0.1553813318411509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,127,0.05716800192991892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,255,0.18134399255116782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,255,0.07612266639868419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,511,0.25169599056243896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,1,0.08932800094286601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,1,0.28942932685216266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,511,0.11757333079973857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,3,0.08968533078829448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,3,0.2905600070953369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,7,0.28993600606918335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,7,0.08980799714724223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,15,0.08955732981363933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,15,0.29634666442871094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,31,0.2977173328399658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,31,0.0897706647713979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,63,0.298250675201416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,63,0.0892693301041921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,127,0.10217600067456563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,127,0.2945173382759094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,1,0.009008000294367472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,1,0.006229333579540253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,3,0.008922666932145754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,7,0.009050666665037474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,3,0.006079999729990959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,7,0.0063040001938740415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,255,0.13809067010879517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,15,0.009141333401203156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,255,0.3482293287913005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,15,0.006341333190600078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,31,0.009759999811649323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,31,0.0063040001938740415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,63,0.011642667154471079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,511,0.47778133551279706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,511,0.21681066354115805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,63,0.005968000118931134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,127,0.0069226666043202085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,127,0.011482667177915573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,1023,0.013450667262077332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,255,0.011557333171367645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,255,0.007957333077987036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,511,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,511,0.008250666782259941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,1023,0.008896000062425932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,2047,0.02160533269246419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,2047,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,4095,0.025114665428797405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,4095,0.012789333860079447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,8191,0.03629333277543386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,8191,0.017637333522240322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,16383,0.05585599939028422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,16383,0.029722665747006733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,32767,0.08456533153851827
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,32767,0.04247466723124186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,65535,0.1362933317820231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,1,0.008997333546479544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,3,0.005882666756709416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,65535,0.07182933390140533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,1,0.0058133335163195925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,3,0.009136000027259191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,7,0.008943999807039896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,7,0.006277333324154218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,15,0.009141333401203156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,15,0.005695999910434087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,131071,0.23479467630386353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,31,0.0099093330403169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,31,0.006074666976928711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,63,0.011391999820868174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,255,0.007760000104705493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,63,0.006064000229040782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,131071,0.1169599990049998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,127,0.011391999820868174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,127,0.006234666953484218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,255,0.012629333883523941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,511,0.013007999708255133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,511,0.008016000191370646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,1023,0.019152000546455383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,1023,0.009472000102202097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,2047,0.01190399999419848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,2047,0.028597332537174225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,4095,0.0405973345041275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,4095,0.01838933303952217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,8191,0.059157331784566246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,8191,0.024773334463437397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,32767,0.13078932960828146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,16383,0.09039466579755147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,16383,0.0429066667954127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,32767,0.0658079981803894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,1,0.010309333602587381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,1,0.00589866687854131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,3,0.0102186668664217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,3,0.006042666733264923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,65535,0.11141332983970642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,65535,0.2222506602605184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,15,0.010341333225369453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,7,0.010368000095089277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,7,0.005877333382765452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,15,0.005882666756709416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,31,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,31,0.005999999741713206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,127,0.006234666953484218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,63,0.010245333115259806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,131071,0.41521600882212323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,63,0.005914666379491488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,127,0.010069333637754122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,131071,0.2023306687672933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,255,0.013290667285521826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,255,0.007743999982873599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,511,0.01947733387351036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,511,0.008890666688481966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,1023,0.028197333216667175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,8191,0.08688533306121826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,1023,0.013584000368913015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,2047,0.0432533323764801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,2047,0.01716800034046173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,4095,0.0687360018491745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,4095,0.026015999416510265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,8191,0.04055999964475632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,16383,0.12914133071899414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,16383,0.06438933312892914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,32767,0.215664009253184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,32767,0.11148266990979512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,1,0.008037333066264788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,1,0.007967999825874964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,65535,0.3931573232014974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,65535,0.20694400866826376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,1,0.005978666866819064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,1,0.005941333249211311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,3,0.007983999947706858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,3,0.0080960001796484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,3,0.005829333638151486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,3,0.006591999903321266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,7,0.007946666950980822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,7,0.006010666489601135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,131071,0.7482773462931315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,7,0.00795199970404307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,131071,0.3975946505864461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,7,0.0059199997534354525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,15,0.007967999825874964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,15,0.008261333530147871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,15,0.00595199999709924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,15,0.006261333202322324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,31,0.0081386665503184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,31,0.008287999778985977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,31,0.006208000083764394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,127,0.009109333157539368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,31,0.00590933362642924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,63,0.008672000219424566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,63,0.009365333244204521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,63,0.005946666623155276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,255,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,255,0.007333333293596904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,63,0.006224000205596288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,127,0.008997333546479544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,127,0.006474666918317477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,127,0.0063146669417619705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,511,0.007706666365265846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,255,0.012448000411192576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,255,0.007429333403706551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,511,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,511,0.011978667229413986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,511,0.007226666435599327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,1023,0.016229332735141117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,1023,0.01701333373785019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,2047,0.007621333623925845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,1023,0.007205333560705185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,1023,0.008687999720374743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,4095,0.03282133241494497
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,2047,0.019573333362738293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,2047,0.024351999163627625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,2047,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,4095,0.02364266663789749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,4095,0.007626666376988093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,4095,0.013445333888133367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,8191,0.027610667049884796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,8191,0.04553066690762838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,8191,0.00766933336853981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,8191,0.022384000321229298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,16383,0.03392533212900162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,16383,0.06954666475454967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,16383,0.030608000854651134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,16383,0.007834666719039282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,32767,0.03491200009981791
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,32767,0.09304533402125041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,32767,0.008143999924262365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,32767,0.05154666801293691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,65535,0.03526933242877325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,65535,0.14063466588656107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,65535,0.00790933333337307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,65535,0.08469866712888081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,1,0.009818666925032934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,1,0.009578666960199675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,1,0.006079999729990959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,1,0.005893333504597346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,3,0.00973866693675518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,3,0.009861333295702934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,3,0.0058133335163195925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,3,0.005834666391213735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,7,0.009775999933481216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,7,0.009690666571259499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,7,0.00578666664659977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,131071,0.03669333209594091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,7,0.00596266674498717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,131071,0.007989333321650824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,131071,0.23918400208155313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,15,0.009850666547815004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,15,0.009648000200589498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,15,0.0059199997534354525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,15,0.0060159998635451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,31,0.009925333162148794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,131071,0.1444586714108785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,31,0.009925333162148794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,31,0.006277333324154218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,31,0.005888000130653381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,63,0.009653333574533463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,63,0.009765333185593287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,63,0.005973333492875099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,63,0.006111999973654747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,127,0.0099093330403169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,255,0.011898666620254517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,127,0.010064000263810158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,127,0.006159999718268712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,127,0.006271999950210254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,255,0.011722666521867117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,255,0.007173333317041397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,255,0.007829333345095316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,511,0.01655999943614006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,511,0.017071999609470367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,511,0.0074986666440963745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,511,0.008181333541870117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,1023,0.021669333179791767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,1023,0.024197332561016083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,2047,0.008234666660428047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,1023,0.007829333345095316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,1023,0.010298666854699453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,2047,0.028175999720891316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,2047,0.0359199990828832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,2047,0.016010666886965435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,4095,0.034245334565639496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,4095,0.052842666705449425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,4095,0.008410666758815447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,4095,0.020645332833131153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,8191,0.03446399917205175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,8191,0.06799466907978058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,8191,0.008277333031098047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,8191,0.028768000503381092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,16383,0.034341332813103996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,16383,0.08932266632715861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,16383,0.008240000034372011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,16383,0.04644800225893656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,32767,0.034559999903043113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,32767,0.008240000034372011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,32767,0.13467199603716531
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,32767,0.07880533238252004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,65535,0.03440000116825104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,65535,0.00821333316465219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,1,0.007743999982873599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,65535,0.14353066682815552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,65535,0.2216800053914388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,1,0.007717333113153775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,1,0.006842666616042455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,1,0.005615999922156334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,3,0.007770666852593422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,3,0.007530666887760162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,3,0.005829333638151486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,3,0.005653333539764087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,7,0.007829333345095316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,7,0.007525333513816197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,7,0.005935999875267346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,7,0.005690666536490123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,15,0.007706666365265846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,15,0.007626666376988093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,15,0.00589866687854131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,15,0.00573333352804184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,31,0.00786666696270307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,31,0.008154666672150293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,131071,0.036101333796978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,31,0.00578666664659977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,31,0.00578666664659977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,63,0.008656000097592672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,63,0.0086666668454806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,63,0.005759999776879947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,63,0.005957333371043205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,127,0.00873066671192646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,127,0.00874133345981439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,127,0.0060159998635451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,127,0.006330666442712148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,131071,0.4001599947611491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,255,0.008767999708652496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,255,0.00879466657837232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,131071,0.00820266641676426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,255,0.006709333509206772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,255,0.007514666765928268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,131071,0.2713013291358948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,511,0.009173333023985228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,511,0.009749333063761393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,511,0.0069386667261521024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,511,0.00855466661353906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,1023,0.013722666849692663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,1023,0.015568000574906668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,1023,0.0069973332186539965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,1023,0.007765333478649457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,2047,0.013077333569526672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,2047,0.016650666793187458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,2047,0.006751999879876773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,2047,0.009066666786869368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,4095,0.013013333082199097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,4095,0.016575999557971954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,4095,0.006725333631038666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,4095,0.01009599988659223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,8191,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,16383,0.006831999868154526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,8191,0.014549333602190018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,8191,0.007162666569153468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,32767,0.014970666418472925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,8191,0.011839999506870905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,32767,0.018629333625237148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,16383,0.01339200014869372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,16383,0.01573866605758667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,16383,0.014325333138306936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,32767,0.00660800002515316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,32767,0.017711999515692394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,65535,0.016602666427691776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,65535,0.02145066608985265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,1,0.007765333478649457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,65535,0.006842666616042455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,65535,0.02204799900452296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,1,0.00590933362642924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,1,0.007541333635648091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,1,0.005653333539764087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,3,0.007749333356817563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,3,0.007584000006318092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,3,0.005664000287652016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,3,0.005712000032265981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,7,0.007541333635648091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,131071,0.028970666229724884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,131071,0.02090666691462199
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,7,0.007696000238259633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,15,0.007727999861041705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,131071,0.029146666328112285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,15,0.006064000229040782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,7,0.005685333162546158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,7,0.005701333284378052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,31,0.005759999776879947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,15,0.007685333490371704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,63,0.008496000121037165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,131071,0.006778666749596596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,15,0.005759999776879947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,31,0.008047999814152718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,31,0.008063999935984612
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,31,0.006026666611433029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,127,0.0063573333124319715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,63,0.008512000242869059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,63,0.005632000043988228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,63,0.005658666913708051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,127,0.0085333331177632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,127,0.008661333471536636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,127,0.00624533308049043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,255,0.008837333569924036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,255,0.008896000062425932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,255,0.006768000001708667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,255,0.007194666812817256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,511,0.008623999853928884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,511,0.009514666472872099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,511,0.006773333375652631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,511,0.0075573331365982694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,1023,0.01333333303531011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,1023,0.015599999576807022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,1023,0.00702400008837382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,1023,0.007957333077987036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,2047,0.013077333569526672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,2047,0.016490666816631954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,2047,0.0069973332186539965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,2047,0.00916800027092298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,4095,0.012645332763592402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,4095,0.013178666432698568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,4095,0.007007999966541926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,4095,0.010037333394090334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,8191,0.014090667168299357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,8191,0.015194666882356008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,8191,0.007098666702707608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,8191,0.012133333832025528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,16383,0.014181333283583323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,16383,0.016010666886965435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,16383,0.006864000111818314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,16383,0.014181333283583323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,32767,0.015253332753976187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,32767,0.019487999379634857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,32767,0.017973333597183228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,32767,0.007071999832987785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,65535,0.022661333282788593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,65535,0.016917333006858826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,65535,0.00684799998998642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,65535,0.022426667312781017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,1,0.007536000261704127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,1,0.0075573331365982694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,1,0.005674666414658229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,1,0.0058026667684316635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,3,0.00754666638871034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,3,0.007589333380262057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,3,0.0058399997651577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,3,0.0058080001423756284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,7,0.007610666876037915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,7,0.007578666632374127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,7,0.0057920000205437345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,131071,0.03984000037113825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,15,0.0075626665105422335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,131071,0.02160533269246419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,7,0.0058026667684316635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,15,0.007776000226537387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,131071,0.0069973332186539965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,15,0.006101333225766818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,31,0.007930666829148928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,15,0.0058186668902635574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,31,0.007994666695594788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,131071,0.032485333581765495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,31,0.005765333150823911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,31,0.0059199997534354525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,63,0.008693333094318708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,63,0.008874666566650072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,63,0.005829333638151486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,63,0.006111999973654747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,127,0.008682666967312494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,127,0.008757333581646284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,127,0.006261333202322324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,127,0.006810666372378667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,255,0.009072000160813332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,255,0.008634666601816813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,255,0.00701333334048589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,255,0.007280000175038974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,511,0.008762666955590248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,511,0.009808000177145004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,511,0.007727999861041705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,511,0.006784000123540561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,1023,0.014767999450365702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,1023,0.015413332730531693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,1023,0.00701333334048589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,1023,0.00814933329820633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,2047,0.013077333569526672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,2047,0.01664000004529953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,2047,0.007040000210205714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,2047,0.008752000207702318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,4095,0.013541333377361298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,4095,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,4095,0.00707733320693175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,16383,0.01421333352724711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,4095,0.01007466639081637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,8191,0.01368533323208491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,8191,0.015925332903862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,8191,0.006858666737874349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,8191,0.011898666620254517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,16383,0.016261332978804905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,32767,0.015541333705186844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,16383,0.0069226666043202085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,16383,0.014298666268587112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,32767,0.01966399947802226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,32767,0.008703999842206636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,32767,0.017829333742459614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,65535,0.01703466723362605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,65535,0.02342933416366577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,65535,0.007247999931375186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,1,0.0075626665105422335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,65535,0.0222080002228419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,1,0.007536000261704127
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,1,0.005834666391213735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,1,0.005664000287652016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,3,0.007823999971151352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,3,0.007685333490371704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,3,0.00589866687854131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,3,0.005770666524767876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,7,0.007776000226537387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,7,0.007770666852593422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,7,0.005888000130653381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,131071,0.03156266609827677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,131071,0.021946666141351063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,131071,0.03973866750796636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,15,0.005904000252485275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,131071,0.007109333450595538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,7,0.0058026667684316635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,15,0.007711999739209811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,15,0.007807999849319458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,15,0.005877333382765452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,31,0.007925333455204964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,31,0.00784533346692721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,63,0.005872000008821487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,31,0.005935999875267346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,31,0.005914666379491488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,63,0.008592000231146812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,63,0.00874133345981439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,63,0.006031999985376994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,127,0.00873066671192646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,127,0.008767999708652496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,127,0.006325333068768184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,511,0.00891733355820179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,127,0.006298666819930077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,255,0.00874133345981439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,255,0.008954666554927826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,255,0.00702400008837382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,255,0.007237333183487256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,1023,0.013647999614477158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,511,0.009749333063761393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,511,0.007135999699433644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,511,0.007733333234985669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,1023,0.01565333331624667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,1023,0.007002666592597961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,1023,0.008000000069538752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,2047,0.012831999609867731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,2047,0.012821332861979803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,2047,0.007002666592597961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,2047,0.00890666681031386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,4095,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,4095,0.01351999988158544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,4095,0.0068800002336502075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,4095,0.009989333028594652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,8191,0.013738666971524557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,8191,0.015696000307798386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,8191,0.007098666702707608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,8191,0.012080000092585882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,16383,0.014767999450365702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,16383,0.017269333203633625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,16383,0.006858666737874349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,16383,0.014309333016475042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,32767,0.026874666412671406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,32767,0.01775466650724411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,32767,0.00706666645904382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,32767,0.019871999820073444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,65535,0.02475200096766154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,65535,0.01978133370478948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,65535,0.007167999943097432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,65535,0.032399999598662056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,1,0.009648000200589498
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,1,0.009754666437705358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,1,0.005775999898711841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,1,0.005781333272655805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,3,0.009935999910036722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,3,0.009754666437705358
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,3,0.005775999898711841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,3,0.00595199999709924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,7,0.009749333063761393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,131071,0.026367999613285065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,7,0.009701333319147428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,7,0.00596266674498717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,7,0.005946666623155276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,131071,0.05682133138179779
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,15,0.009984000275532404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,131071,0.0069333333522081375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,15,0.009893333539366722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,15,0.0057386669019858045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,15,0.005946666623155276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,31,0.00985599992175897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,31,0.010197333370645842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,63,0.005957333371043205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,31,0.005968000118931134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,63,0.006122666721542676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,131071,0.03771200031042099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,31,0.006042666733264923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,63,0.009722666814923286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,63,0.00955200009047985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,127,0.009786666681369146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,127,0.009882666791478792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,127,0.006213333457708359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,127,0.006330666442712148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,255,0.012133333832025528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,255,0.011909333368142446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,255,0.007061333085099856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,255,0.007360000163316727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,511,0.016496000190575916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,511,0.01692266638080279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,511,0.00730666642387708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,511,0.0085333331177632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,1023,0.02128533273935318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,1023,0.02439466615517934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,1023,0.007813333223263422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,1023,0.010101333260536194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,2047,0.016336000214020412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,2047,0.02733866622050603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,2047,0.035989334185918175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,2047,0.008053333188096682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,4095,0.03421333432197571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,4095,0.05266666909058889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,4095,0.008101333553592363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,4095,0.0206986665725708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,8191,0.034058667719364166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,8191,0.06831466654936473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,8191,0.00820266641676426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,8191,0.028912000358104706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,16383,0.03453333427508672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,16383,0.008309333274761835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,16383,0.08982400099436443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,16383,0.046309332052866616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,32767,0.034346667428811394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,32767,0.008410666758815447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,32767,0.13555733362833658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,32767,0.07826666533946991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,1,0.010159999753038088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,1,0.006229333579540253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,1,0.010133333504199982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,1,0.006128000095486641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,3,0.010314666976531347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,3,0.010112000008424124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,3,0.006159999718268712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,3,0.006165333092212677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,7,0.010154666379094124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,7,0.01022933361430963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,7,0.006218666831652324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,7,0.006282666698098183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,15,0.010288000106811523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,15,0.010224000240365664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,15,0.005914666379491488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,65535,0.03439466655254364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,15,0.006031999985376994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,31,0.010090666512648264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,31,0.009898666913310686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,65535,0.008447999755541483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,31,0.0063146669417619705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,31,0.006021333237489064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,63,0.009850666547815004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,63,0.010197333370645842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,63,0.006048000107208888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,63,0.006208000083764394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,127,0.009893333539366722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,127,0.010224000240365664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,65535,0.22277865807215372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,127,0.006815999746322632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,127,0.006309333567818006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,255,0.0183146670460701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,255,0.01794133335351944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,65535,0.1430346667766571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,255,0.007600000128149986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,255,0.008405333384871483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,511,0.022645334402720135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,511,0.024879999458789825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,1023,0.03286399940649668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,511,0.00820266641676426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,511,0.009525333220760027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,1023,0.03972800076007843
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,1023,0.008746666833758354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,1023,0.016010666886965435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,2047,0.03514666606982549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,2047,0.008469333251317343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,2047,0.05171733101209005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,2047,0.019802667200565338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,4095,0.035546667873859406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,4095,0.06712000072002411
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,4095,0.008757333581646284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,4095,0.02802666773398717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,8191,0.0351946676770846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,8191,0.08959466218948364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,8191,0.00850133349498113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,8191,0.0450133333603541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,16383,0.13297067085901895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,16383,0.035674666364987694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,16383,0.008693333094318708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,16383,0.07684266567230225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,32767,0.0351946676770846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,32767,0.00854399986565113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,32767,0.14082666238149008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,1,0.007802666475375493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,32767,0.22251200675964355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,1,0.00595199999709924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,1,0.007610666876037915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,1,0.005679999788602193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,3,0.007818666597207388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,3,0.007584000006318092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,3,0.005717333406209946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,7,0.007674666742483775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,3,0.00590933362642924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,7,0.0075626665105422335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,7,0.005664000287652016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,7,0.00596266674498717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,15,0.00786666696270307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,15,0.007594666754206021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,15,0.005674666414658229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,15,0.005946666623155276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,31,0.007840000092983246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,65535,0.03576533248027166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,31,0.00786666696270307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,31,0.0059199997534354525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,31,0.005888000130653381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,63,0.008725333337982496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,65535,0.2677759925524394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,63,0.008442666381597519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,65535,0.3978346586227417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,63,0.005824000264207522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,127,0.008901333436369896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,65535,0.008986666798591614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,63,0.006069333602984746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,127,0.00867733359336853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,127,0.006074666976928711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,127,0.006224000205596288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,255,0.00873066671192646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,255,0.008986666798591614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,255,0.007093333328763644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,255,0.007226666435599327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,1023,0.013909333695967993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,511,0.008752000207702318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,511,0.01003200002014637
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,511,0.007034666836261749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,511,0.007605333502093951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,1023,0.015743999431530636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,1023,0.00696000022192796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,1023,0.008309333274761835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,2047,0.01321600005030632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,2047,0.013973332941532135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,2047,0.007034666836261749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,2047,0.008976000050703684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,4095,0.01022933361430963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,8191,0.014554666976133982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,8191,0.007141333073377609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,4095,0.014080000420411428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,8191,0.011909333368142446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,4095,0.01443733274936676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,4095,0.0069333333522081375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,8191,0.01709866647919019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,16383,0.015392000476519266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,16383,0.01802666609485944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,16383,0.00721066693464915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,16383,0.014602666099866232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,32767,0.018485333770513535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,32767,0.020117333779732387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,32767,0.007061333085099856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,32767,0.02758399893840154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,65535,0.02072000006834666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,65535,0.03404266635576884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,65535,0.007055999711155891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,65535,0.024127999941507976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,1,0.007690666864315669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,1,0.007690666864315669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,1,0.0058133335163195925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,1,0.007520000139872233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,3,0.007727999861041705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,3,0.007807999849319458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,3,0.0058080001423756284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,3,0.005829333638151486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,7,0.007797333101431529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,7,0.007765333478649457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,7,0.005941333249211311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,7,0.005994666367769241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,15,0.007829333345095316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,15,0.007749333356817563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,131071,0.027269333600997925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,15,0.005877333382765452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,131071,0.007184000064929326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,131071,0.03721066564321518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,131071,0.05835733314355215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,15,0.0058399997651577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,31,0.008058666562040647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,31,0.008047999814152718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,31,0.005829333638151486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,31,0.005893333504597346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,63,0.008687999720374743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,63,0.008885333314538002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,63,0.005999999741713206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,63,0.005957333371043205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,127,0.009125333279371262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,127,0.008853333070874214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,127,0.006058666855096817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,127,0.006144000217318535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,255,0.008842666943868002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,255,0.009050666665037474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,255,0.007125333572427432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,255,0.007349333415428798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,511,0.00903466654320558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,511,0.009872000043590864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,511,0.00726400005320708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,2047,0.013701333353916803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,511,0.007594666754206021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,1023,0.0116799995303154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,1023,0.011813333878914515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,1023,0.00701333334048589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,1023,0.008197333042820295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,2047,0.013914667069911957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,2047,0.00696000022192796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,4095,0.0102186668664217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,2047,0.009354666496316591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,4095,0.013893333574136099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,4095,0.01479999969402949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,4095,0.0069333333522081375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,8191,0.016970666746298473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,8191,0.02143999934196472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,8191,0.007018666714429855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,8191,0.013125333935022354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,16383,0.01785600061217944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,16383,0.025199999411900837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,16383,0.007221333061655362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,16383,0.015941333025693893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,32767,0.02165333429972331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,32767,0.03682666768630346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,32767,0.022431999444961548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,32767,0.007301333049933116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,65535,0.025722667574882507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,65535,0.05683733522891998
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,65535,0.007189333438873291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,65535,0.033946665624777474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,1,0.01020800011853377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,1,0.009952000031868616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,1,0.0060159998635451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,1,0.006133333469430606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,3,0.009973333527644476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,3,0.010149333626031876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,3,0.006224000205596288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,3,0.005957333371043205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,7,0.010010666524370512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,7,0.010351999973257383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,7,0.006021333237489064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,131071,0.03271466741959254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,7,0.005999999741713206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,131071,0.060645331939061485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,15,0.010239999741315842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,131071,0.0075519997626543045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,131071,0.09097599983215332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,31,0.010154666379094124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,15,0.009935999910036722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,31,0.006325333068768184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,15,0.006026666611433029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,15,0.005989333614706993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,31,0.010239999741315842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,63,0.011343999455372492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,31,0.006042666733264923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,63,0.01033599985142549
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,63,0.006389333556095759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,63,0.005957333371043205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,255,0.017727999637524288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,127,0.010064000263810158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,127,0.010266666611035665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,127,0.006277333324154218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,127,0.00624533308049043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,255,0.017973333597183228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,255,0.007402666533986728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,255,0.007893333211541176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,511,0.02274666726589203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,1023,0.03292799989382426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,1023,0.039488000174363456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,511,0.024959998826185863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,511,0.008250666782259941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,2047,0.03572266548871994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,511,0.009402666861812273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,1023,0.008447999755541483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,1023,0.016224000602960587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,2047,0.05170666674772898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,2047,0.008623999853928884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,2047,0.019893333315849304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,4095,0.03510399907827377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,4095,0.06744533280531566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,4095,0.008458666503429413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,4095,0.027818667391935985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,8191,0.03533866753180822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,8191,0.08981866637865703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,8191,0.008736000085870424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,8191,0.04548799991607666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,16383,0.03518400092919668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,16383,0.13411200046539307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,16383,0.008618666479984919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,16383,0.07740266621112823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,1,0.014186666657527288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,1,0.014416000495354334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,1,0.0063786668082078295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,1,0.006282666698098183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,3,0.014208000153303146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,3,0.01434133326013883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,3,0.006405333057045937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,3,0.006351999938488007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,7,0.014159999787807465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,7,0.014368000129858652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,32767,0.03547733277082443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,7,0.006405333057045937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,7,0.006293333445986112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,15,0.014085333794355392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,15,0.006400000303983688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,15,0.014181333283583323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,15,0.006341333190600078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,32767,0.22411733865737915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,31,0.01414399966597557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,31,0.01458666721979777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,32767,0.008645333349704742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,31,0.006511999915043513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,32767,0.14200533429781595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,31,0.0063573333124319715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,63,0.014101333916187286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,63,0.01481066644191742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,63,0.0063733334342638654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,63,0.006549333532651265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,127,0.0069759997228781385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,127,0.014122666170199713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,127,0.013893333574136099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,127,0.006629333520929019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,255,0.026714667677879333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,255,0.026687999566396076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,255,0.008282666405042013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,255,0.009482666850090027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,511,0.03601066768169403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,511,0.040181333820025124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,511,0.009893333539366722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,511,0.015696000307798386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,1023,0.0363520011305809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,1023,0.04924799998601278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,2047,0.03846399982770284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,1023,0.010191999996701876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,1023,0.019413333386182785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,2047,0.010128000130256018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,2047,0.06842133402824402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,2047,0.027818667391935985
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,4095,0.03845333307981491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,4095,0.09035199880599976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,4095,0.009925333162148794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,4095,0.04475200176239014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,8191,0.03868799904982249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,8191,0.1348426640033722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,8191,0.009925333162148794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,8191,0.07662400106589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,16383,0.038506666819254555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,16383,0.2234399914741516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,16383,0.009898666913310686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,16383,0.14153066277503967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,1,0.01421333352724711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,1,0.01421333352724711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,1,0.006394666930039723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,1,0.006261333202322324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,3,0.014192000031471252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,3,0.014192000031471252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,3,0.006229333579540253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,32767,0.03836799909671148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,3,0.007189333438873291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,7,0.01431999976436297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,7,0.014175999909639359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,7,0.006144000217318535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,32767,0.009925333162148794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,7,0.006453333422541618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,15,0.014197333405415217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,15,0.014101333916187286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,15,0.006128000095486641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,15,0.006341333190600078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,31,0.014277332772811254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,32767,0.4002240101496379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,31,0.014122666170199713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,31,0.006170666466156642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,31,0.0064319999267657595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,63,0.014453332871198654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,63,0.014181333283583323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,63,0.006186666587988536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,63,0.00613866684337457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,32767,0.26977066198984784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,127,0.01421333352724711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,127,0.014453332871198654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,127,0.00666133314371109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,255,0.026191999514897663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,127,0.006709333509206772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,255,0.026202666262785595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,255,0.008325333396593729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,255,0.009434666484594345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,511,0.03999999910593033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,511,0.03585600107908249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,511,0.010090666512648264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,1023,0.03595199932654699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,511,0.01553600033124288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,1023,0.049312000473340355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,1023,0.010053333515922228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,1023,0.019573333362738293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,2047,0.038191998998324074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,2047,0.06809600194295247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,2047,0.010064000263810158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,2047,0.027829334139823914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,4095,0.03818666686614355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,4095,0.09006399909655254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,4095,0.009962666779756546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,4095,0.044810667634010315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,1,0.02250666668017705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,1,0.022240000466505688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,8191,0.038245332737763725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,1,0.007280000175038974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,8191,0.010245333115259806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,8191,0.13532800475756326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,1,0.006890666360656421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,3,0.02272533377011617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,3,0.006789333497484525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,3,0.022229333718617756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,7,0.022522665560245514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,8191,0.07696533203125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,7,0.0069386667261521024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,3,0.00697066696981589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,7,0.022639999787012737
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,7,0.00707733320693175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,15,0.022197333474953968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,15,0.022682666778564453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,15,0.00701333334048589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,15,0.0069973332186539965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,31,0.022682666778564453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,31,0.022416000564893086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,31,0.00701333334048589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,31,0.006800000245372455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,63,0.022469334304332733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,63,0.02231466770172119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,63,0.006965333595871925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,63,0.007061333085099856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,127,0.022410665949185688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,127,0.02254933367172877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,127,0.007776000226537387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,127,0.00795199970404307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,255,0.043290664752324425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,255,0.04307200014591217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,255,0.01358933374285698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,255,0.015365333606799444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,511,0.04334400097529093
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,511,0.05194666484991709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,511,0.013450667262077332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,1023,0.0432533323764801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,511,0.019498666127522785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,1023,0.06630933284759521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,1023,0.01358933374285698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,1023,0.02812266598145167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,2047,0.045824001232783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,2047,0.013594667116800943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,2047,0.09284800291061401
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,2047,0.04524266719818115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,4095,0.045461331804593406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,4095,0.013770667215188345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,4095,0.1360373298327128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,4095,0.07672533392906189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,1,0.02298133323589961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,1,0.022704000274340313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,8191,0.013530666629473368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,8191,0.04612799982229868
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,1,0.006949333474040031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,8191,0.22887466351191202
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,1,0.007104000076651573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,8191,0.1406719982624054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,3,0.022704000274340313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,3,0.022698665658632915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,3,0.00707733320693175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,3,0.007146666447321574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,7,0.02279466638962428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,7,0.022895999252796173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,7,0.0069866664707660675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,7,0.007109333450595538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,15,0.02272533377011617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,15,0.022789334257443745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,15,0.007157333195209503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,15,0.007184000064929326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,31,0.022917332748572033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,31,0.022661333282788593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,31,0.00707733320693175
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,31,0.00684799998998642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,63,0.022677332162857056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,63,0.02260799954334895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,63,0.007125333572427432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,63,0.007040000210205714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,127,0.02242133269707362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,127,0.02271466702222824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,127,0.008176000167926153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,127,0.008000000069538752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,255,0.04303466777006785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,255,0.04271466533342997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,255,0.013722666849692663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,255,0.01534933348496755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,511,0.04274666806062063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,511,0.052613332867622375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,511,0.013541333377361298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,511,0.019391999890406925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,1023,0.042912001411120095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,1023,0.06623466809590657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,1023,0.013733333597580591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,1023,0.02787200113137563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,2047,0.04524266719818115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,2047,0.09272000193595886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,2047,0.013717333475748697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,2047,0.04497600098450979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,1,0.038618666430314384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,4095,0.045461331804593406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,4095,0.13715199629465738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,1,0.03909866760174433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,1,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,1,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,3,0.03875199953715006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,4095,0.07691200077533722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,4095,0.013733333597580591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,3,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,3,0.03886399914820989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,3,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,7,0.0388373335202535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,7,0.03860266755024592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,7,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,7,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,15,0.038912000755469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,15,0.0391893337170283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,15,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,15,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,31,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,31,0.03863999992609024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,31,0.03890133400758108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,31,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,63,0.03884266565243403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,63,0.038693333665529885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,63,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,63,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,127,0.03821333249409994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,127,0.03858133405447006
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,127,0.012655999511480331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,127,0.012949333836634954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,255,0.04083733260631561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,255,0.040133332212766014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,255,0.012714666624863943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,255,0.016373333831628162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,511,0.04081066697835922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,511,0.054485330979029335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,511,0.012842666357755661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,511,0.025146665672461193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,1023,0.040933333337306976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,1023,0.07726933558781941
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,1023,0.012613333761692047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,1023,0.04242133100827535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,2047,0.04438933233420054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,2047,0.12524799505869547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,2047,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,2047,0.07388266424338023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,1,0.007834666719039282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,1,0.008112000301480293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,1,0.005648000165820122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,1,0.00590933362642924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,4095,0.04402133325735728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,3,0.008170666793982187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,3,0.007765333478649457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,3,0.005973333492875099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,3,0.006069333602984746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,4095,0.21414399147033691
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,4095,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,7,0.007717333113153775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,7,0.007882666463653246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,7,0.005925333127379417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,7,0.005999999741713206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,4095,0.1386613349119822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,15,0.007791999727487564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,15,0.00589866687854131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,15,0.007861333588759104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,15,0.006458666796485583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,31,0.007994666695594788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,31,0.007989333321650824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,31,0.006085333103934924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,31,0.0057386669019858045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,127,0.00890666681031386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,63,0.008687999720374743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,63,0.008682666967312494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,63,0.005904000252485275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,63,0.00589866687854131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,127,0.00890666681031386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,127,0.0063146669417619705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,127,0.006010666489601135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,255,0.00898133342464765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,255,0.008799999952316284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,255,0.007109333450595538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,1023,0.011887999872366587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,255,0.007514666765928268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,511,0.00903466654320558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,511,0.009994666402538618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,511,0.007130666946371396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,511,0.007567999884486198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,1023,0.011674666156371435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,2047,0.007184000064929326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,1023,0.00701333334048589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,1023,0.008309333274761835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,2047,0.013637332866589228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,4095,0.007109333450595538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,2047,0.014250667144854864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,2047,0.00921066664159298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,4095,0.01393066719174385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,4095,0.014469332993030548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,4095,0.010117333382368088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,8191,0.01692266638080279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,8191,0.021456000705560047
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,8191,0.007082666580875714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,8191,0.013242666920026144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,16383,0.018138666947682697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,16383,0.01588800052801768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,16383,0.02492266645034154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,16383,0.007007999966541926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,32767,0.03622400015592575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,32767,0.02139200021823247
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,32767,0.007301333049933116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,32767,0.021770666042963665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,65535,0.007226666435599327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,65535,0.02532800038655599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,65535,0.05630933245023092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,65535,0.033173332611719765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,1,0.007893333211541176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,1,0.007813333223263422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,1,0.006085333103934924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,1,0.005904000252485275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,3,0.007802666475375493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,3,0.00810666692753633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,3,0.00595199999709924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,3,0.005834666391213735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,7,0.008047999814152718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,7,0.007925333455204964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,7,0.005674666414658229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,7,0.006053333481152852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,15,0.007946666950980822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,15,0.007658666620651881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,15,0.006069333602984746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,15,0.006031999985376994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,31,0.007914666707317034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,31,0.008309333274761835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,31,0.00600533311565717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,131071,0.03249066571394602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,31,0.007029333462317784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,131071,0.007573333258430163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,63,0.00890666681031386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,131071,0.09175466497739156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,63,0.00901333304742972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,127,0.009423999736706415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,63,0.005930666501323382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,63,0.005701333284378052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,127,0.009098666409651438
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,131071,0.05927466849486033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,127,0.006341333190600078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,255,0.00898133342464765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,127,0.006288000072042148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,255,0.008890666688481966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,255,0.006917333230376244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,255,0.007743999982873599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,511,0.011493333925803503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,511,0.011391999820868174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,511,0.007146666447321574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,511,0.0075519997626543045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,1023,0.011530666301647821
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,1023,0.011557333171367645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,1023,0.00706666645904382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,1023,0.008101333553592363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,2047,0.01589866727590561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,2047,0.018592000007629395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,2047,0.00721066693464915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,2047,0.009765333185593287
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,4095,0.016373333831628162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,4095,0.020373333245515823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,4095,0.00706666645904382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,4095,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,8191,0.019882666567961376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,8191,0.028101332485675812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,8191,0.007258666679263115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,8191,0.014933332800865173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,16383,0.03854399919509888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,16383,0.023423999547958374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,16383,0.007258666679263115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,16383,0.019823999454577763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,32767,0.007280000175038974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,32767,0.02902399996916453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,32767,0.03440000116825104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,32767,0.062352001667022705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,65535,0.09380267063776652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,65535,0.007690666864315669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,65535,0.0351946676770846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,65535,0.0517493337392807
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,1,0.039408000806967415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,1,0.039861333866914116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,1,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,1,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,131071,0.03728533287843069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,131071,0.007754666730761528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,3,0.03952533255020777
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,3,0.03984533250331879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,131071,0.14711999893188477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,131071,0.09302399555842082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,7,0.03921599934498469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,3,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,3,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,7,0.03948266555865606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,7,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,7,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,15,0.039818666875362396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,15,0.039733332892258964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,15,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,15,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,31,0.039664000272750854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,31,0.039493332306543984
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,31,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,31,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,63,0.03947199881076813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,63,0.0391839991013209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,63,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,127,0.039066667358080544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,63,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,127,0.038949333131313324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,127,0.012789333860079447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,127,0.013440000514189402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,255,0.04101866732041041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,255,0.04121600091457367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,255,0.012671999633312225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,255,0.01676799977819125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,511,0.041349334021409355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,511,0.054527997970581055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,511,0.012719999998807907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,511,0.02534399926662445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,1023,0.04131199916203817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,1023,0.07799999912579854
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,1023,0.012768000364303589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,1023,0.0422986646493276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,2047,0.04507199923197428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,1,0.07165333131949107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,2047,0.12548800309499106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,2047,0.012784000486135483
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,1,0.07113066812356313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,1,0.018229333062966663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,2047,0.07400533556938171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,1,0.0185759998857975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,3,0.07161599894364674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,3,0.07121600210666656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,3,0.018453333526849747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,7,0.07086400190989177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,3,0.018464000274737675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,7,0.01838933303952217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,7,0.07096000015735626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,7,0.01852799952030182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,15,0.07123200098673503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,15,0.07134399811426799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,15,0.018320000420014065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,15,0.018453333526849747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,31,0.07124266525109609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,31,0.07102400064468384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,31,0.018522666146357853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,31,0.018277333428462345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,63,0.07047999898592631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,63,0.07008533179759979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,63,0.01852799952030182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,63,0.01842133328318596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,127,0.0705386648575465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,127,0.07070399820804596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,127,0.021381333470344543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,127,0.021695998807748158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,255,0.07796800136566162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,255,0.07694399853547414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,255,0.0216799999276797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,255,0.030293333033720653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,511,0.07784000039100647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,511,0.09847467144330342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,511,0.021407999098300934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,511,0.047050664822260536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,1023,0.07785599927107494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,1023,0.14193600416183472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,1023,0.021674667795499165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,1023,0.07807999849319458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,2047,0.08142933249473572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,2047,0.021365332106749218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,2047,0.23361599445343018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,2047,0.14029866456985474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,float16,1,0.07237866520881653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,1,0.07202133536338806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,fp8,1,0.01850133389234543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,1,0.01850133389234543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,float16,3,0.0718560020128886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,3,0.07189333438873291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,fp8,3,0.018458666900793713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,3,0.018613333503405254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,float16,7,0.07196266452471416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,fp8,7,0.01855466639002164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,7,0.07187200089295705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,7,0.01846933364868164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,float16,15,0.07172266642252605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,15,0.0718506673971812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,fp8,15,0.018464000274737675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,15,0.018533332894245785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,float16,31,0.07198399802049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,31,0.0724533349275589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,fp8,31,0.018538666268189747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,31,0.01841066653529803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,float16,63,0.07094933092594147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,63,0.07108800113201141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,fp8,63,0.018453333526849747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,63,0.01848000039656957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,float16,127,0.07201600074768066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,127,0.07268266876538594
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,fp8,127,0.021530665457248688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,127,0.021690666675567627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,float16,255,0.07809600234031677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,255,0.07788266738255818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,fp8,255,0.02165333429972331
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,255,0.030597334106763203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,511,0.09890133142471313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,fp8,511,0.021583999196688335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,511,0.047194664676984154
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,float16,1023,0.07790400087833405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,1023,0.14364799857139587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,fp8,1023,0.021877333521842957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,float16,1,0.13238933682441711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,1,0.13239999612172446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,1023,0.07895466685295105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,fp8,1,0.032229334115982056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,1,0.03230933348337809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,float16,3,0.13249599933624268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,3,0.133242666721344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,fp8,3,0.032186667124430336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,3,0.032672000428040825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,float16,7,0.1325973371664683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,7,0.1332266628742218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,fp8,7,0.032474666833877563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,7,0.03253333270549774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,float16,15,0.1323466698328654
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,15,0.13342932860056558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,fp8,15,0.0323786661028862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,15,0.03235200047492981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,float16,31,0.1323946714401245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,31,0.13320533434549967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,fp8,31,0.032245332996050514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,31,0.03216533362865448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,float16,63,0.13342400391896567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,63,0.13332800070444742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,fp8,63,0.03217600037654241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,63,0.03234133372704188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,float16,127,0.13540800412495932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,127,0.13462400436401367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,fp8,127,0.03997333347797394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,float16,511,0.07770666480064392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,127,0.0405173326532046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,float16,255,0.1450933317343394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,255,0.1453973352909088
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,fp8,255,0.04114133367935816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,255,0.0572266678015391
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,float16,511,0.1455573340257009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,511,0.1841119925181071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,fp8,511,0.041146665811538696
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,1,0.00789866658548514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,511,0.08653333783149719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,1,0.007989333321650824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,1,0.006010666489601135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,1,0.005872000008821487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,3,0.007850666840871176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,3,0.007887999837597212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,3,0.005984000240763028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,3,0.0058613332609335584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,7,0.007914666707317034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,7,0.007887999837597212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,7,0.006010666489601135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,7,0.00590933362642924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,15,0.008069333309928576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,15,0.007893333211541176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,1023,0.27374400695164997
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,15,0.005984000240763028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,15,0.00590933362642924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,float16,1023,0.14619200428326926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,31,0.00821333316465219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,31,0.008026666939258575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,1023,0.14878933628400168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,fp8,1023,0.040933333337306976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,31,0.005946666623155276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,31,0.005877333382765452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,63,0.008682666967312494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,63,0.00874133345981439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,63,0.00600533311565717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,63,0.005973333492875099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,127,0.008954666554927826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,127,0.00891733355820179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,127,0.0063786668082078295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,127,0.006144000217318535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,255,0.009088000282645226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,255,0.009050666665037474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,255,0.007055999711155891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,255,0.007424000029762586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,1023,0.007226666435599327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,511,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,511,0.011589333415031433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,511,0.007162666569153468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,511,0.007514666765928268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,1023,0.011503999431928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,1023,0.011717333147923151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,1023,0.008042666440208754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,2047,0.016095999628305435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,2047,0.018351999421914417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,2047,0.007184000064929326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,8191,0.019893333315849304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,2047,0.009679999823371569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,4095,0.016634666671355564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,8191,0.015008000036080679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,4095,0.020981334149837494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,4095,0.007247999931375186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,4095,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,8191,0.027984000742435455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,8191,0.007482666522264481
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,16383,0.022970666488011677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,16383,0.007429333403706551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,16383,0.0394400010506312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,16383,0.019882666567961376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,32767,0.028725333511829376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,32767,0.061648001273473106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,32767,0.007541333635648091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,32767,0.03440533330043157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,65535,0.03550933301448822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,65535,0.09508267045021057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,65535,0.007903999959429106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,65535,0.05150400102138519
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,1,0.007920000081261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,1,0.005989333614706993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,1,0.0080960001796484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,1,0.005946666623155276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,3,0.007936000203092894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,3,0.0081386665503184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,3,0.007610666876037915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,3,0.005925333127379417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,7,0.008143999924262365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,7,0.00789866658548514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,7,0.005914666379491488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,7,0.006170666466156642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,15,0.008816000074148178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,15,0.00786666696270307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,15,0.00624533308049043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,15,0.005999999741713206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,31,0.008037333066264788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,31,0.008261333530147871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,31,0.006021333237489064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,31,0.005882666756709416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,131071,0.03709866603215536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,63,0.008703999842206636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,131071,0.14682666460673013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,63,0.006709333509206772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,63,0.008976000050703684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,127,0.008810666700204214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,63,0.005989333614706993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,127,0.00884799969693025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,131071,0.007813333223263422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,127,0.006570666407545407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,127,0.006288000072042148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,255,0.011343999455372492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,255,0.007786666974425316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,131071,0.09351467092831929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,255,0.011338666081428528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,255,0.0069919998447100324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,511,0.011503999431928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,511,0.011653333902359009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,511,0.00697066696981589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,511,0.008117333054542542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,1023,0.01701333373785019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,1023,0.017130666722853977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,2047,0.00790933333337307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,2047,0.010543999572594961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,1023,0.007370666911204656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,1023,0.008853333070874214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,2047,0.01904533306757609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,2047,0.023813332120577495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,4095,0.02369600037733714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,4095,0.032613334556420646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,4095,0.007360000163316727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,4095,0.013477332890033722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,8191,0.027589333554108936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,8191,0.044480000933011375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,8191,0.007530666887760162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,8191,0.022954667607943218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,16383,0.034154665966828666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,16383,0.06844800213972728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,16383,0.007834666719039282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,16383,0.03070933371782303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,32767,0.03475199888149897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,32767,0.09299733241399129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,32767,0.007957333077987036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,32767,0.0507893313964208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,65535,0.03462400039037069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,65535,0.13925333817799887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,1,0.009226666763424873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,65535,0.007850666840871176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,1,0.006261333202322324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,3,0.009029333169261614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,65535,0.08296533425649007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,3,0.005904000252485275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,7,0.00890666681031386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,7,0.006101333225766818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,15,0.009343999748428663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,15,0.005850666513045629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,31,0.00985599992175897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,31,0.006021333237489064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,63,0.011461333682139715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,63,0.0058559998869895935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,127,0.011653333902359009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,127,0.006602666651209195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,255,0.012554666648308435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,255,0.007584000006318092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,511,0.008090666805704435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,511,0.013082666943470636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,131071,0.007936000203092894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,1023,0.019120000302791595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,131071,0.14619200428326926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,1023,0.00943999985853831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,131071,0.24014933904012045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,2047,0.028463999430338543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,131071,0.036288000643253326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,2047,0.012015999605258306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,4095,0.04116799930731455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,4095,0.01844266677896182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,8191,0.05990933378537496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,8191,0.024314666787783306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,16383,0.09177600344022115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,16383,0.04258666435877482
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,1,0.010384000216921171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,1,0.00590933362642924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,32767,0.1304639975229899
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,32767,0.06568000217278798
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,3,0.010405333091815313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,3,0.0058666666348775225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,7,0.010239999741315842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,7,0.005893333504597346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,15,0.010122666756312052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,15,0.006069333602984746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,31,0.01009599988659223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,31,0.005968000118931134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,63,0.010213333492477735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,63,0.005946666623155276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,65535,0.11168533563613892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,127,0.0100426667680343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,127,0.006170666466156642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,65535,0.22181866566340128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,255,0.013232000172138214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,255,0.007877333089709282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,511,0.019178666174411774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,511,0.008581333483258883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,1023,0.028170667588710785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,1023,0.013973332941532135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,2047,0.016949333250522614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,2047,0.04353066782156626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,4095,0.06772799789905548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,4095,0.026026666164398193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,8191,0.08694400389989217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,8191,0.04008000095685323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,16383,0.12798399726549783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,16383,0.06418666740258534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,131071,0.41513065497080487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,131071,0.20363734165827432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,32767,0.21478933095932007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,32767,0.11161067088445027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,1,0.008672000219424566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,1,0.005717333406209946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,3,0.005968000118931134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,3,0.008570666735370954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,7,0.008661333471536636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,7,0.00595199999709924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,65535,0.2058133284250895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,15,0.00877333308259646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,15,0.005695999910434087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,65535,0.3906773328781128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,31,0.009434666484594345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,31,0.005882666756709416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,63,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,63,0.005893333504597346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,127,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,127,0.006735999758044879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,255,0.007429333403706551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,255,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,511,0.015263999501864115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,511,0.007829333345095316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,1023,0.016362667083740234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,1023,0.008229333286484083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,2047,0.016597333053747814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,2047,0.009066666786869368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,4095,0.01658133293191592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,4095,0.010351999973257383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,8191,0.018415999909241993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,8191,0.012458667159080505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,16383,0.01941866676012675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,16383,0.014405333747466406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,32767,0.02333866556485494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,32767,0.018485333770513535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,1,0.008682666967312494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,131071,0.7441973686218262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,1,0.005717333406209946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,3,0.008656000097592672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,3,0.005728000154097875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,131071,0.39633599917093915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,7,0.008672000219424566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,7,0.005690666536490123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,15,0.008874666566650072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,65535,0.023573334018389385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,15,0.005615999922156334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,31,0.009429333110650381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,31,0.0057920000205437345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,65535,0.02848000079393387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,63,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,63,0.005717333406209946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,127,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,127,0.006079999729990959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,255,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,255,0.007514666765928268
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,511,0.015173333386580149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,1023,0.008330666770537695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,511,0.0075519997626543045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,1023,0.016085332880417507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,4095,0.015439999600251516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,131071,0.03873066604137421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,2047,0.016085332880417507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,2047,0.00921066664159298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,4095,0.010101333260536194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,131071,0.03014400104681651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,8191,0.018805333723624546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,8191,0.01228800043463707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,16383,0.02006400004029274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,16383,0.014767999450365702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,32767,0.018346666047970455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,32767,0.02457600086927414
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,1,0.008538666491707167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,1,0.005648000165820122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,3,0.008602666358153025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,3,0.00589866687854131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,7,0.008394666636983553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,7,0.005946666623155276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,15,0.008778666456540426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,15,0.005573333551486333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,31,0.009477333476146063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,63,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,31,0.005850666513045629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,65535,0.023402666052182514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,65535,0.030767999589443207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,63,0.005957333371043205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,127,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,127,0.00625599982837836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,255,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,255,0.007301333049933116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,511,0.015466666469971338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,511,0.007674666742483775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,1023,0.016293333222468693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,1023,0.008181333541870117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,2047,0.016229332735141117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,131071,0.0618399977684021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,2047,0.009952000031868616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,4095,0.015439999600251516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,4095,0.010512000570694605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,8191,0.019146667172511418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,131071,0.03461333364248276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,8191,0.01227733368674914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,16383,0.020090666910012562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,16383,0.014853333433469137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,32767,0.024442667762438457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,32767,0.01854933301607768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,1,0.00867733359336853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,1,0.005695999910434087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,3,0.0059199997534354525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,3,0.008645333349704742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,7,0.008581333483258883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,7,0.005946666623155276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,15,0.008810666700204214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,15,0.005664000287652016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,65535,0.03075733284155528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,65535,0.02384000023206075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,31,0.009621333330869675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,31,0.0058666666348775225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,127,0.011338666081428528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,63,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,63,0.005722666780153911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,127,0.006048000107208888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,255,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,255,0.007610666876037915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,511,0.015285332997639975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,131071,0.03357866654793421
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,131071,0.061247999469439186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,511,0.00761600024998188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,1023,0.016421332955360413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,1023,0.008352000266313553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,2047,0.015194666882356008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,2047,0.009093333035707474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,4095,0.0161013330022494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,4095,0.010474666953086853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,8191,0.0194560003777345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,8191,0.012522666404644648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,16383,0.022069332500298817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,16383,0.015008000036080679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,32767,0.03543466577927271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,32767,0.02054399996995926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,1,0.010458666831254959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,1,0.006133333469430606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,3,0.010234666367371878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,3,0.006074666976928711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,7,0.010298666854699453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,15,0.0102613332370917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,7,0.005968000118931134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,15,0.006048000107208888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,65535,0.05114666620890299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,65535,0.025920001169045765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,31,0.010239999741315842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,31,0.006026666611433029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,63,0.010512000570694605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,127,0.010112000008424124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,63,0.00596266674498717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,127,0.006341333190600078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,255,0.013269333789745966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,255,0.007978666573762894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,511,0.019023999571800232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,511,0.008821333448092142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,1023,0.027749332288901012
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,1023,0.014010666559139887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,2047,0.04363200068473816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,2047,0.017317333569129307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,131071,0.08338133494059245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,4095,0.06794666747252147
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,4095,0.026554666459560394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,131071,0.04105599969625473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,8191,0.08802666266759236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,8191,0.04134399940570196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,1,0.010709332923094431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,16383,0.1290720005830129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,16383,0.06573333342870076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,1,0.006186666587988536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,3,0.010442666709423065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,3,0.006159999718268712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,32767,0.21699732542037964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,7,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,7,0.005978666866819064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,32767,0.11246400078137715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,15,0.010672000547250112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,15,0.005797333394487699
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,31,0.0106133334338665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,31,0.00590933362642924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,63,0.010490667074918747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,63,0.006058666855096817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,127,0.0103946669648091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,127,0.0063680000603199005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,255,0.021061333517233532
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,255,0.008463999877373377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,511,0.02831999957561493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,511,0.01301866645614306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,65535,0.2079306642214457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,1023,0.046762665112813316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,1023,0.019482667247454327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,2047,0.06638399759928386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,65535,0.3959093491236369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,2047,0.025402667621771496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,4095,0.08531199892361958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,4095,0.03980266551176707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,8191,0.1281706690788269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,8191,0.06404800216356914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,1,0.00871999996403853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,16383,0.21569067239761353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,16383,0.1106666624546051
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,1,0.005722666780153911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,3,0.0059199997534354525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,3,0.008698666468262672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,7,0.00873066671192646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,15,0.008816000074148178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,7,0.0058133335163195925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,32767,0.3922186692555745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,15,0.005941333249211311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,31,0.009557333464423815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,63,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,32767,0.2057173252105713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,31,0.0058613332609335584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,63,0.005712000032265981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,127,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,127,0.006351999938488007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,255,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,255,0.0074879998962084455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,511,0.015306666493415833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,511,0.00790933333337307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,1023,0.01643199970324834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,1023,0.008522666369875273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,2047,0.015775999675194424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,2047,0.009018666421373686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,4095,0.017103999853134155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,4095,0.01051733394463857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,65535,0.3946719964345296
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,8191,0.02083733429511388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,8191,0.012437333663304647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,65535,0.7470560073852539
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,16383,0.02252800017595291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,16383,0.014943999548753103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,32767,0.037018666664759316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,32767,0.02111999938885371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,1,0.00879466657837232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,1,0.0058399997651577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,3,0.008757333581646284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,3,0.005877333382765452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,7,0.008661333471536636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,7,0.0059199997534354525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,15,0.008885333314538002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,65535,0.02624000112215678
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,15,0.006224000205596288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,31,0.009546666716535887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,63,0.0058399997651577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,65535,0.053690666953722634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,127,0.006250666454434395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,31,0.005957333371043205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,63,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,127,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,255,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,255,0.007584000006318092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,511,0.015552000453074774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,131071,0.042208001017570496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,131071,0.0851146678129832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,511,0.008021333565314611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,1023,0.013429333766301474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,1023,0.008320000022649765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,2047,0.01676799977819125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,2047,0.009450666606426239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,4095,0.018160000443458557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,4095,0.010522666076819101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,8191,0.026901334524154663
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,8191,0.014186666657527288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,16383,0.03289066751797994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,16383,0.017125333348910015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,32767,0.054058666030565895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,32767,0.023823998868465424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,1,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,1,0.006181333214044571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,3,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,3,0.006128000095486641
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,7,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,7,0.005989333614706993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,65535,0.0811359981695811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,15,0.010709332923094431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,65535,0.04877333343029022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,15,0.006122666721542676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,31,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,31,0.005973333492875099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,63,0.00600533311565717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,63,0.010634666929642359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,127,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,127,0.006565333033601443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,255,0.021141332884629566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,511,0.02812800059715907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,255,0.00877333308259646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,131071,0.06769600013891856
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,511,0.013605333864688873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,1023,0.04651199777921041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,1023,0.020015999674797058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,131071,0.13778666655222574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,2047,0.06681066751480103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,2047,0.02603200078010559
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,4095,0.08668266733487447
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,4095,0.04036800066630045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,8191,0.12839999794960022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,8191,0.06459199885527293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,1,0.015333333363135656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,1,0.006528000036875407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,3,0.015365333606799444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,3,0.006442666674653689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,7,0.015253332753976187
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,7,0.006511999915043513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,16383,0.22080000241597494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,15,0.015344000111023584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,16383,0.11308800180753072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,15,0.006522666662931442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,31,0.015311999867359797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,31,0.006400000303983688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,63,0.015498666713635126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,63,0.006453333422541618
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,127,0.015263999501864115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,127,0.007344000041484833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,255,0.030154667794704437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,255,0.013327999661366144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,511,0.04696533580621084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,511,0.019226666539907455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,1023,0.06275199850400288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,1023,0.025114665428797405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,2047,0.08585600058237712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,32767,0.40281065305074054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,2047,0.03976000100374222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,32767,0.20712000131607056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,4095,0.1280693312486013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,4095,0.06400000055631001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,8191,0.11243733763694763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,8191,0.22011200586954752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,1,0.015344000111023584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,1,0.006458666796485583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,3,0.015295999745527903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,3,0.006517333288987477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,16383,0.399893323580424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,7,0.01553600033124288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,16383,0.20543466011683145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,7,0.006624000146985054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,15,0.015274666249752045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,15,0.006442666674653689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,31,0.015578666081031164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,31,0.006762666627764702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,63,0.015354666858911514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,63,0.006469333544373512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,127,0.015509333461523056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,127,0.0074506668994824094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,255,0.013738666971524557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,255,0.030565333863099415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,511,0.0466186652580897
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,511,0.019727999965349834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,1023,0.0622026671965917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,1023,0.025744001070658367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,2047,0.08685333530108134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,2047,0.04041066765785217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,32767,0.3975573380788167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,4095,0.12945600350697836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,32767,0.760213295618693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,1,0.024959998826185863
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,4095,0.06396799782911937
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,1,0.009706666693091393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,3,0.025045332809289295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,3,0.009679999823371569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,8191,0.22244266668955484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,7,0.02513066679239273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,7,0.009642666826645533
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,15,0.025077333052953083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,8191,0.11200533310572307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,15,0.009786666681369146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,31,0.025066666305065155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,31,0.009818666925032934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,63,0.009546666716535887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,63,0.024986666937669117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,127,0.02510933329661687
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,127,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,255,0.050383999943733215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,255,0.019189332922299702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,511,0.06506666541099548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,1023,0.04041599979003271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,511,0.0252960001428922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,1023,0.0839413305123647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,2047,0.13195199767748514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,2047,0.06419200201829274
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,1,0.025450666745503742
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,4095,0.2227519949277242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,3,0.025370667378107708
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,4095,0.11144000291824341
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,1,0.009808000177145004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,3,0.009888000165422758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,8191,0.20666666825612387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,8191,0.4068479935328166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,7,0.00978133330742518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,7,0.02531733363866806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,15,0.02533866713444392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,31,0.025279998779296875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,15,0.009866666669646898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,31,0.009941333283980688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,63,0.009808000177145004
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,63,0.02480533222357432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,127,0.02534399926662445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,127,0.011482667177915573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,255,0.05072533090909322
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,255,0.019904000063737232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,511,0.06540800134340923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,511,0.025920001169045765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,1023,0.08430400490760803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,1023,0.0408693328499794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,1,0.043653334180514015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,2047,0.13150399923324585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,2047,0.06481599807739258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,1,0.013861333330472311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,3,0.043882668018341064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,3,0.01392000044385592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,7,0.013973332941532135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,7,0.044256001710891724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,15,0.04399999976158142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,4095,0.2267626722653707
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,15,0.013733333597580591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,4095,0.11142399907112122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,31,0.04384533564249674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,63,0.04358933369318644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,31,0.013834666460752487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,63,0.013674666484196981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,127,0.04428266485532125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,127,0.01581866666674614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,255,0.05460800230503082
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,255,0.02145066608985265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,511,0.07368533313274384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,511,0.03693866729736328
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,1,0.008757333581646284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,1023,0.11900800466537476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,1,0.0058613332609335584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,1023,0.06080533564090729
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,3,0.008639999975760778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,3,0.0058080001423756284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,7,0.008709333216150602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,15,0.009589333087205887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,7,0.005834666391213735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,15,0.0058026667684316635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,2047,0.21634133656819662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,2047,0.10798399647076924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,31,0.009573333586255709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,31,0.005759999776879947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,63,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,63,0.005904000252485275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,127,0.011306667079528173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,4095,0.40531734625498456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,511,0.007941333577036858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,4095,0.20246932903925577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,127,0.006261333202322324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,255,0.011349332829316458
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,1023,0.00850133349498113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,255,0.0075040000180403394
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,511,0.015589332828919092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,1023,0.01328533391157786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,2047,0.016261332978804905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,2047,0.009461333354314169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,8191,0.013957332819700241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,4095,0.017898666361967724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,4095,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,8191,0.02661866694688797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,16383,0.03259200106064478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,16383,0.016730666160583496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,32767,0.05433600147565206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,1,0.009018666421373686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,32767,0.02421333392461141
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,1,0.005994666367769241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,3,0.00891733355820179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,3,0.0058026667684316635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,7,0.008965333302815756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,7,0.0058399997651577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,15,0.008965333302815756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,15,0.006069333602984746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,31,0.00972800018886725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,31,0.0059199997534354525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,63,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,65535,0.0817386656999588
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,65535,0.04950400193532308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,63,0.005877333382765452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,127,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,255,0.011343999455372492
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,127,0.006170666466156642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,255,0.007402666533986728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,511,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,511,0.007749333356817563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,1023,0.01340266689658165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,1023,0.008629333227872849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,2047,0.02130666623512904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,2047,0.01032533310353756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,4095,0.024688000480333965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,4095,0.011877333124478659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,131071,0.13566399614016214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,131071,0.06796266635258992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,8191,0.03622400015592575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,16383,0.027162666122118633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,8191,0.015493333339691162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,16383,0.055349335074424744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,32767,0.08353599905967712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,32767,0.039877332746982574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,1,0.04427733520666758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,65535,0.13703999916712442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,1,0.013658666362365087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,65535,0.06931200126806895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,3,0.043935999274253845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,3,0.013973332941532135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,7,0.045093332727750145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,7,0.013786666095256805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,15,0.0444106658299764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,15,0.014245333770910898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,131071,0.11441066861152649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,31,0.044266665975252785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,31,0.013877333452304205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,131071,0.23809067408243814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,63,0.04386133452256521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,63,0.013621332744757334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,127,0.045093332727750145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,127,0.01632000009218852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,255,0.05493866900602976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,255,0.02276800076166789
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,511,0.03779733429352442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,511,0.07470933099587758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,1023,0.12006933490435283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,1,0.08063999811808269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,1023,0.061333333452542625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,1,0.023573334018389385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,3,0.08060800035794576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,3,0.023584000766277313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,2047,0.21715732415517172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,7,0.08004799981911977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,7,0.023584000766277313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,2047,0.10903466741243999
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,15,0.08027733365694682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,15,0.02350933353106181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,31,0.08046400050322215
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,31,0.02359466751416524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,63,0.023365333676338196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,127,0.0828000009059906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,127,0.02937600016593933
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,255,0.09658666451772054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,511,0.13244799772898355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,511,0.06468800206979115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,1023,0.21526400248209634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,1023,0.11077866951624553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,2047,0.4031146764755249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,1,0.023925334215164185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,2047,0.20263999700546265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,1,0.08097599943478902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,3,0.08074666559696198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,3,0.02366400013367335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,7,0.08062933385372162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,7,0.02380266785621643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,15,0.08100800216197968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,15,0.023578666150569916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,31,0.02384000023206075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,63,0.08132266501585643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,63,0.023743999501069386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,127,0.08343467116355896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,127,0.030576000610987347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,255,0.09715732932090759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,255,0.04270400106906891
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,511,0.1336426635583242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,511,0.0655680000782013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,63,0.08055466910203297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,1,0.14965867002805075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,1,0.0467199981212616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,1023,0.11170132954915364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,1023,0.2188106576601664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,255,0.04195733368396759
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,3,0.14914133151372275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,3,0.04586133360862732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,7,0.15025599797566733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,7,0.04730666677157084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,15,0.04678399860858917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,15,0.15059733390808105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,31,0.15213867028554282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,31,0.04630400240421295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,63,0.0469706654548645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,63,0.15271466970443726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,31,0.08078399797280629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,127,0.15492799878120422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,127,0.054645334680875145
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,255,0.18173867464065552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,1,0.00873066671192646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,255,0.07402133444945018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,1,0.0058613332609335584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,3,0.008762666955590248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,3,0.005882666756709416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,7,0.008762666955590248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,15,0.008896000062425932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,7,0.00595199999709924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,15,0.0058133335163195925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,511,0.24816532929738364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,31,0.009632000078757605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,31,0.0058080001423756284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,511,0.11596266428629558
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,63,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,63,0.0057386669019858045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,127,0.011365332951148352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,127,0.006293333445986112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,255,0.011488000551859537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,255,0.007642666498819987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,511,0.012831999609867731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,1023,0.40370134512583417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,511,0.00790933333337307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,1023,0.013317332913478216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,1023,0.00842666688064734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,2047,0.02165866643190384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,1023,0.20083733399709067
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,2047,0.010415999839703241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,8191,0.03642666588226954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,4095,0.0249493345618248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,4095,0.011936000237862269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,8191,0.016095999628305435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,16383,0.027866666515668232
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,16383,0.05565333366394043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,1,0.009002666920423508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,1,0.005824000264207522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,3,0.00892800030608972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,3,0.00595199999709924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,7,0.008826666822036108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,32767,0.040634666879971824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,32767,0.08534399668375652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,15,0.009061333412925402
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,7,0.00600533311565717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,15,0.005984000240763028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,31,0.009813333551088968
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,31,0.005994666367769241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,63,0.011312000453472137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,127,0.006277333324154218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,65535,0.1362933317820231
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,255,0.012661332885424295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,255,0.007765333478649457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,63,0.007130666946371396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,65535,0.06989333530267079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,127,0.01139733319481214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,511,0.008069333309928576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,511,0.013269333789745966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,1023,0.019178666174411774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,1023,0.009279999881982803
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,2047,0.028399998943010967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,2047,0.011621333658695221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,4095,0.04152533411979675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,4095,0.01871466636657715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,131071,0.11440533399581909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,8191,0.06001600126425425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,131071,0.23322133223215738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,8191,0.024271999796231587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,16383,0.08847467104593913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,16383,0.042175998290379844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,1,0.007994666695594788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,1,0.00785600021481514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,32767,0.06564799944559734
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,32767,0.1313813328742981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,1,0.006101333225766818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,1,0.006048000107208888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,3,0.00786666696270307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,3,0.007994666695594788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,3,0.006751999879876773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,3,0.006053333481152852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,7,0.007994666695594788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,65535,0.11049600442250569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,7,0.007994666695594788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,7,0.006159999718268712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,7,0.006064000229040782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,65535,0.22259734074274698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,15,0.007983999947706858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,15,0.007711999739209811
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,15,0.006122666721542676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,31,0.0081386665503184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,15,0.005968000118931134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,31,0.006133333469430606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,31,0.008261333530147871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,31,0.005824000264207522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,63,0.008810666700204214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,63,0.008656000097592672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,63,0.006186666587988536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,63,0.006074666976928711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,127,0.008879999940594038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,127,0.00897066667675972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,127,0.0064853330453236895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,131071,0.20206934213638306
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,127,0.0063786668082078295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,255,0.011802667131026586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,131071,0.4147786696751912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,255,0.011381333072980246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,255,0.007184000064929326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,255,0.007327999919652939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,511,0.011616000284751257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,511,0.011887999872366587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,511,0.007189333438873291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,511,0.007642666498819987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,1023,0.01613333324591319
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,1023,0.016677333662907284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,2047,0.024330665667851765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,2047,0.0075093333919843035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,1023,0.008890666688481966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,2047,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,1023,0.007231999809543292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,2047,0.01904533306757609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,4095,0.02369600037733714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,4095,0.03252800057331721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,4095,0.007189333438873291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,4095,0.0136266661187013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,8191,0.027776000400384266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,8191,0.04514666895071665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,8191,0.007680000116427739
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,8191,0.021984001000722248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,16383,0.034117333590984344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,16383,0.06892266869544983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,16383,0.007978666573762894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,16383,0.030917334059874218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,32767,0.0925600032011668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,32767,0.035349334279696144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,32767,0.051221330960591636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,32767,0.007813333223263422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,65535,0.0349386657277743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,65535,0.13979199528694153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,65535,0.008117333054542542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,1,0.0075519997626543045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,1,0.007733333234985669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,1,0.005744000275929769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,1,0.005679999788602193
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,65535,0.08399466673533122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,3,0.007754666730761528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,3,0.007653333246707916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,3,0.005722666780153911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,3,0.0058453331391016645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,7,0.007610666876037915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,7,0.007674666742483775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,7,0.005754666402935982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,7,0.005941333249211311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,15,0.007637333124876022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,15,0.005685333162546158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,15,0.008053333188096682
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,15,0.005904000252485275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,31,0.00789866658548514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,31,0.00795199970404307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,31,0.006010666489601135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,31,0.006762666627764702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,63,0.008565333361426989
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,63,0.008496000121037165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,63,0.0060159998635451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,63,0.006026666611433029
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,127,0.008762666955590248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,127,0.008687999720374743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,127,0.006090666477878888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,127,0.006330666442712148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,255,0.008816000074148178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,255,0.008682666967312494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,255,0.006911999856432279
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,255,0.007167999943097432
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,511,0.008810666700204214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,511,0.009679999823371569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,511,0.006789333497484525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,511,0.007717333113153775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,131071,0.03694933404525121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,131071,0.1472586691379547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,131071,0.007850666840871176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,1023,0.013269333789745966
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,1023,0.015439999600251516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,131071,0.23836266994476318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,1023,0.006949333474040031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,2047,0.016314666718244553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,2047,0.012650666137536367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,1023,0.007887999837597212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,2047,0.0069440001000960665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,2047,0.00922133338948091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,4095,0.016538667182127636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,8191,0.013125333935022354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,4095,0.012805332740147909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,4095,0.006735999758044879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,4095,0.0100426667680343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,8191,0.014890667051076889
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,8191,0.00701333334048589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,8191,0.011978667229413986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,16383,0.013823999712864557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,16383,0.01565333331624667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,16383,0.006805333619316419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,16383,0.014021333307027817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,32767,0.015381333728631338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,32767,0.0068853336075941724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,32767,0.01893866683046023
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,32767,0.017759999881188076
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,65535,0.01670933390657107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,65535,0.022111999491850536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,65535,0.006837333242098491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,65535,0.022448000808556873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,1,0.0075093333919843035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,1,0.007589333380262057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,1,0.0058186668902635574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,1,0.005706666658322017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,3,0.007610666876037915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,3,0.007701333612203598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,3,0.005882666756709416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,3,0.0057920000205437345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,7,0.007717333113153775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,7,0.0075626665105422335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,7,0.0058026667684316635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,7,0.0058453331391016645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,131071,0.02956266701221466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,131071,0.02146133283774058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,15,0.0074453335255384445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,15,0.007530666887760162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,31,0.008016000191370646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,15,0.005973333492875099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,15,0.0058399997651577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,31,0.007733333234985669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,131071,0.006784000123540561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,63,0.008576000109314919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,31,0.005669333040714264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,31,0.005882666756709416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,127,0.008826666822036108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,63,0.00855466661353906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,63,0.005749333028992017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,131071,0.03044266750415166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,63,0.005829333638151486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,127,0.008634666601816813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,127,0.0063680000603199005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,127,0.00620266670982043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,255,0.008805333326260248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,255,0.008618666479984919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,255,0.00701333334048589
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,1023,0.013365333278973898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,255,0.007189333438873291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,511,0.008538666491707167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,511,0.009824000298976898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,511,0.00702400008837382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,511,0.007376000285148621
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,1023,0.01523200049996376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,1023,0.007109333450595538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,1023,0.008165333420038223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,2047,0.013162666310866674
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,2047,0.016314666718244553
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,2047,0.00702400008837382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,2047,0.008896000062425932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,4095,0.013056000073750814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,4095,0.013151999562978745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,4095,0.006949333474040031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,4095,0.010144000252087912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,8191,0.013882666826248169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,8191,0.015210667004187902
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,16383,0.016389333953460056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,16383,0.0068693334857622785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,8191,0.006842666616042455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,8191,0.012069333344697952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,16383,0.014416000495354334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,16383,0.01434133326013883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,32767,0.019402666638294857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,32767,0.016063999384641647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,32767,0.006858666737874349
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,32767,0.01811733345190684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,65535,0.017322666943073273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,65535,0.022869333624839783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,65535,0.007087999954819679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,65535,0.02258666604757309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,1,0.00979200005531311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,1,0.009866666669646898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,1,0.005904000252485275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,1,0.0059199997534354525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,3,0.009786666681369146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,3,0.009797333429257074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,3,0.005978666866819064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,3,0.006064000229040782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,7,0.009722666814923286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,7,0.009818666925032934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,7,0.006085333103934924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,7,0.0058613332609335584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,131071,0.021962667504946392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,131071,0.04053866614898046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,131071,0.0069759997228781385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,15,0.00984533317387104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,15,0.009770666559537252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,15,0.005877333382765452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,31,0.009914666414260864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,15,0.005941333249211311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,31,0.010005333150426546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,31,0.0060159998635451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,131071,0.03275733441114426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,63,0.00620266670982043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,31,0.006111999973654747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,63,0.009679999823371569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,127,0.006298666819930077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,127,0.006277333324154218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,63,0.009866666669646898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,63,0.006053333481152852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,127,0.00978133330742518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,127,0.009882666791478792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,255,0.01201066623131434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,255,0.012128000458081564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,255,0.007173333317041397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,255,0.007520000139872233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,1023,0.021503999829292297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,511,0.016362667083740234
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,511,0.017136000096797943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,511,0.007525333513816197
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,511,0.008378666515151659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,1023,0.02463999887307485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,2047,0.03605333218971888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,1023,0.007690666864315669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,2047,0.008357333640257517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,1023,0.010234666367371878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,2047,0.02756800005833308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,4095,0.034314667185147606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,2047,0.016069332758585613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,4095,0.05310933291912079
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,4095,0.008367999767263731
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,4095,0.020741333564122517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,8191,0.03436800092458725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,8191,0.06789333124955495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,8191,0.008309333274761835
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,8191,0.028864001234372456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,16383,0.03467733412981033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,16383,0.008421333506703377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,16383,0.09101333220799764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,16383,0.04590400060017904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,32767,0.034474665919939675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,32767,0.13482667009035745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,32767,0.008538666491707167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,32767,0.0792799989382426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,65535,0.034645333886146545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,65535,0.22367467482884726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,1,0.007813333223263422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,1,0.007573333258430163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,1,0.00589866687854131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,1,0.0058559998869895935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,65535,0.008320000022649765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,65535,0.14204800128936768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,3,0.007754666730761528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,3,0.007696000238259633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,3,0.005941333249211311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,3,0.0059199997534354525
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,7,0.007786666974425316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,7,0.007584000006318092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,7,0.00590933362642924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,7,0.006021333237489064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,15,0.007770666852593422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,15,0.00766933336853981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,15,0.0058453331391016645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,15,0.006042666733264923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,31,0.007941333577036858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,31,0.007861333588759104
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,31,0.00590933362642924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,31,0.006031999985376994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,63,0.008853333070874214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,63,0.0084906667470932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,63,0.005872000008821487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,63,0.006037333359320958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,127,0.00878399983048439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,127,0.008661333471536636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,127,0.006117333347598712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,127,0.0063040001938740415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,255,0.008687999720374743
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,255,0.008805333326260248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,255,0.0069386667261521024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,255,0.007370666911204656
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,511,0.009136000027259191
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,511,0.009679999823371569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,511,0.006954666847983996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,511,0.007685333490371704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,1023,0.013471999516089758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,1023,0.015578666081031164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,1023,0.00721066693464915
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,1023,0.008122666428486506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,2047,0.013642666240533194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,2047,0.014005333185195923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,2047,0.0068853336075941724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,2047,0.008752000207702318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,4095,0.01421333352724711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,4095,0.014298666268587112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,4095,0.006826666494210561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,4095,0.010266666611035665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,131071,0.03619733452796936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,8191,0.01461333284775416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,131071,0.401962677637736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,8191,0.016789333273967106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,8191,0.011936000237862269
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,8191,0.007104000076651573
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,131071,0.27162667115529376
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,16383,0.015376000354687372
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,131071,0.008357333640257517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,16383,0.017850667238235474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,16383,0.007216000308593114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,16383,0.0143306665122509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,32767,0.019082666685183842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,32767,0.027477333943049114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,32767,0.006965333595871925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,32767,0.0201706662774086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,65535,0.02088533341884613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,65535,0.033999999364217125
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,65535,0.007093333328763644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,65535,0.024319998919963837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,1,0.010255999863147736
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,1,0.0102613332370917
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,1,0.005882666756709416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,1,0.006154666965206464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,3,0.010224000240365664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,3,0.009818666925032934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,3,0.006042666733264923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,3,0.00624533308049043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,7,0.009829333052039146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,7,0.010037333394090334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,131071,0.02773333340883255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,7,0.006197333335876465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,7,0.005850666513045629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,131071,0.05845866600672404
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,15,0.010101333260536194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,15,0.01020800011853377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,15,0.006197333335876465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,131071,0.007247999931375186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,15,0.006042666733264923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,31,0.010170666500926018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,31,0.010330666477481524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,31,0.006250666454434395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,31,0.006224000205596288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,131071,0.03726933399836222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,63,0.010106666634480158
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,63,0.010101333260536194
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,63,0.006154666965206464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,63,0.006335999816656113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,127,0.006159999718268712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,127,0.010037333394090334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,127,0.0100426667680343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,127,0.006448000048597653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,255,0.017946666727463405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,255,0.01800000046690305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,255,0.007429333403706551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,511,0.02258666604757309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,255,0.008016000191370646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,511,0.02565866708755493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,511,0.008357333640257517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,511,0.00943999985853831
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,1023,0.03306133300065994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,1023,0.0397173340121905
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,1023,0.008714666590094566
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,2047,0.03531199942032496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,1023,0.016063999384641647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,2047,0.051781331499417625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,2047,0.00877333308259646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,2047,0.0199946661790212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,4095,0.035461333890755974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,4095,0.06775466601053874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,4095,0.008629333227872849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,4095,0.027727998793125153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,8191,0.03525333354870478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,8191,0.09004799524943034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,8191,0.0086666668454806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,8191,0.04491733511288961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,16383,0.0354666660229365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,16383,0.1341546674569448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,16383,0.008752000207702318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,16383,0.07692799965540568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,32767,0.00867733359336853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,32767,0.035258665680885315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,32767,0.2238453427950541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,32767,0.14059733351071677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,1,0.01431999976436297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,1,0.014240000396966934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,1,0.006037333359320958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,1,0.006533333410819371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,3,0.014352000008026758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,3,0.013861333330472311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,3,0.006384000182151794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,3,0.0063786668082078295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,7,0.013936000565687815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,7,0.014010666559139887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,7,0.006319999694824219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,7,0.006053333481152852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,15,0.014090667168299357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,15,0.0141546664138635
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,15,0.006095999851822853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,15,0.00624533308049043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,31,0.0143306665122509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,31,0.014175999909639359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,31,0.006319999694824219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,31,0.006293333445986112
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,63,0.014010666559139887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,63,0.01394133393963178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,63,0.0063040001938740415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,63,0.006170666466156642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,127,0.01393066719174385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,127,0.014165333161751429
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,65535,0.03528533379236857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,127,0.006586666529377301
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,127,0.006784000123540561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,255,0.02645866572856903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,255,0.02638400097688039
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,255,0.008432000254591307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,255,0.009365333244204521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,511,0.03603733330965042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,511,0.04029333343108495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,511,0.009898666913310686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,511,0.015471999843915304
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,1023,0.03595733394225439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,65535,0.3981333176294963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,65535,0.26791999737421673
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,1023,0.04948799808820089
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,1023,0.009925333162148794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,1023,0.01942933350801468
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,65535,0.008725333337982496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,2047,0.038431999584039055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,2047,0.06841066479682922
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,2047,0.009872000043590864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,2047,0.027445333699385326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,4095,0.03854399919509888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,4095,0.09054399530092876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,4095,0.010144000252087912
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,4095,0.04513599971930186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,8191,0.038373333712418876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,8191,0.1349546710650126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,8191,0.010128000130256018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,8191,0.07638399799664815
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,1,0.022426667312781017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,1,0.022431999444961548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,1,0.006874666859706243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,16383,0.038560000558694206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,1,0.0069866664707660675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,3,0.022389332453409832
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,16383,0.22572267055511475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,3,0.02248000105222066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,3,0.0069386667261521024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,3,0.007034666836261749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,16383,0.010181333248813948
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,7,0.02235200007756551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,7,0.022597332795461018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,7,0.0069333333522081375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,7,0.006895999734600385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,15,0.022474666436513264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,16383,0.14019733667373657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,15,0.02271999915440877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,15,0.007018666714429855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,15,0.00690133310854435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,31,0.022405333817005157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,31,0.0223786657055219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,31,0.0069919998447100324
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,31,0.007205333560705185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,63,0.022463999688625336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,63,0.022111999491850536
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,63,0.0069386667261521024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,127,0.022287999590237934
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,63,0.006773333375652631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,127,0.022304000953833263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,127,0.008031999692320824
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,127,0.008117333054542542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,255,0.04322666426499685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,255,0.04264000058174133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,255,0.013530666629473368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,255,0.01524266724785169
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,511,0.042768001556396484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,511,0.05190933247407278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,511,0.013471999516089758
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,511,0.019695999721686046
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,1023,0.04320000112056732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,1023,0.06574933230876923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,1023,0.01349866638580958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,1023,0.02770666778087616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,2047,0.045509333411852516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,2047,0.013503999759753546
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,2047,0.09278933207194011
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,2047,0.045461331804593406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,4095,0.04553066690762838
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,4095,0.013408000270525614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,4095,0.1363200048605601
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,4095,0.07645333309968312
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,1,0.007653333246707916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,1,0.00784533346692721
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,1,0.005664000287652016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,1,0.0058613332609335584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,3,0.007936000203092894
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,3,0.007663999994595845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,3,0.005914666379491488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,3,0.006074666976928711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,7,0.007605333502093951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,7,0.007685333490371704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,7,0.006037333359320958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,7,0.005824000264207522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,15,0.007663999994595845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,15,0.009093333035707474
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,8191,0.04542933404445648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,15,0.005759999776879947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,15,0.005930666501323382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,31,0.0080960001796484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,31,0.007791999727487564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,31,0.005946666623155276
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,31,0.0063786668082078295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,63,0.008576000109314919
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,63,0.009749333063761393
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,8191,0.013536000003417334
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,63,0.005957333371043205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,8191,0.2281493345896403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,63,0.006085333103934924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,127,0.006197333335876465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,8191,0.1418346663316091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,127,0.008789333204428354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,127,0.00895999992887179
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,127,0.0064213331788778305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,255,0.008778666456540426
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,255,0.008858666444818178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,255,0.007071999832987785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,255,0.00749333327015241
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,511,0.009082666908701261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,511,0.009685333197315535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,511,0.007242666557431221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,1023,0.008453333129485449
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,511,0.007690666864315669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,1023,0.011968000481526056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,1023,0.011957333733638128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,1023,0.007151999821265538
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,2047,0.013557333499193192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,2047,0.014256000518798828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,2047,0.007189333438873291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,2047,0.00901333304742972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,4095,0.014256000518798828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,4095,0.015008000036080679
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,4095,0.007178666690985362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,4095,0.010314666976531347
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,8191,0.01700266698996226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,8191,0.02186666677395503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,8191,0.007120000198483467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,16383,0.007098666702707608
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,8191,0.013317332913478216
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,16383,0.01798933371901512
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,16383,0.02516266703605652
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,16383,0.01591466615597407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,32767,0.021962667504946392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,32767,0.036650667587916054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,32767,0.006895999734600385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,32767,0.022069332500298817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,65535,0.025637333591779072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,65535,0.056943997740745544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,65535,0.03270400067170461
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,65535,0.0075573331365982694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,1,0.03889599939187368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,1,0.03897066662708918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,1,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,1,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,131071,0.03302400062481562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,3,0.03890133400758108
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,3,0.03899733225504557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,131071,0.09241599837938945
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,131071,0.007418666655818622
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,3,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,3,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,131071,0.059578667084376015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,7,0.03915733347336451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,7,0.038933334251244865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,7,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,7,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,15,0.039146666725476585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,15,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,15,0.038736000657081604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,15,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,31,0.0391146664818128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,31,0.03884266565243403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,31,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,31,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,63,0.038693333665529885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,63,0.0383840004603068
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,63,0.010645333677530289
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,63,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,127,0.03853866706291834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,127,0.03850133220354716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,127,0.01268799975514412
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,127,0.012602667013804117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,255,0.040591999888420105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,255,0.04063999901215235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,255,0.012815999488035837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,255,0.016575999557971954
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,511,0.04035733391841253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,511,0.054746667544047035
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,511,0.012725333372751871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,511,0.024879999458789825
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,1023,0.040762667854626976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,1023,0.07760000228881836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,1023,0.012671999633312225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,1023,0.04236799975236257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,2047,0.12489066521326701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,2047,0.04400533437728882
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,2047,0.012576000144084295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,2047,0.07389866809050243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,1,0.0713919997215271
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,4095,0.044138665000597634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,1,0.07083199918270111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,4095,0.21306665738423666
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,4095,0.012805332740147909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,1,0.01825599993268649
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,4095,0.1377120018005371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,1,0.01844266677896182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,3,0.07086400190989177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,3,0.07127999762694041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,3,0.01826133330663045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,3,0.018559999763965607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,7,0.07117333511511485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,7,0.07096000015735626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,7,0.01836799954374631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,7,0.018325333793958027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,15,0.07136533161004384
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,15,0.07100800176461537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,15,0.01848000039656957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,31,0.0710399995247523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,15,0.018383999665578205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,31,0.07109333574771881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,31,0.018383999665578205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,31,0.01823466643691063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,63,0.07045866549015045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,63,0.07017600039641063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,63,0.01859733338157336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,63,0.018272000054518383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,127,0.07071466743946075
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,127,0.07025066514809926
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,127,0.021520001192887623
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,127,0.021327999730904896
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,255,0.07829866806666057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,255,0.07727999985218048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,255,0.02164799968401591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,255,0.029893333713213604
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,511,0.0774293343226115
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,511,0.0971999963124593
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,511,0.021418665846188862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,511,0.046949331959088646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,1023,0.07774400214354198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,1023,0.14226133624712625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,1,0.00790933333337307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,1023,0.02179733415444692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,1023,0.07834666470686595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,1,0.007893333211541176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,1,0.005872000008821487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,1,0.006037333359320958
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,3,0.007829333345095316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,3,0.006074666976928711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,3,0.008000000069538752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,3,0.0064853330453236895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,7,0.007754666730761528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,7,0.008037333066264788
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,7,0.006101333225766818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,15,0.007791999727487564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,7,0.00595199999709924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,15,0.007920000081261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,15,0.00600533311565717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,15,0.005850666513045629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,2047,0.08083199958006541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,2047,0.23323732614517212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,31,0.00814933329820633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,31,0.008223999912540117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,31,0.006048000107208888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,31,0.00590933362642924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,2047,0.021354667842388153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,63,0.00877333308259646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,2047,0.1413226624329885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,127,0.009088000282645226
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,127,0.006277333324154218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,63,0.00892800030608972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,127,0.006384000182151794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,63,0.00613866684337457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,63,0.005882666756709416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,127,0.008799999952316284
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,255,0.007605333502093951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,511,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,255,0.009018666421373686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,255,0.007285333548982938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,255,0.008912000184257826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,511,0.011525332927703857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,511,0.007071999832987785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,511,0.007786666974425316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,1023,0.011333333949247995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,1023,0.011770666887362799
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,1023,0.007141333073377609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,1023,0.008176000167926153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,2047,0.0162773331006368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,2047,0.0184906671444575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,2047,0.007194666812817256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,2047,0.009658666948477427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,4095,0.016719999412695568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,4095,0.02063999945918719
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,4095,0.007205333560705185
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,4095,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,8191,0.01960533360640208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,8191,0.028607999285062153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,8191,0.007034666836261749
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,16383,0.02333866556485494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,8191,0.014858666807413101
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,16383,0.007130666946371396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,16383,0.03851199895143509
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,16383,0.020090666910012562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,32767,0.028922667105992634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,32767,0.007690666864315669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,32767,0.062037333846092224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,32767,0.034117333590984344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,65535,0.035402665535608925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,65535,0.007701333612203598
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,65535,0.05213333169619242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,65535,0.0946720043818156
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,1,0.008832000195980072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,1,0.005904000252485275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,3,0.008816000074148178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,3,0.005968000118931134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,7,0.008863999818762144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,7,0.0058666666348775225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,15,0.009018666421373686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,15,0.006064000229040782
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,31,0.009632000078757605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,31,0.005999999741713206
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,63,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,63,0.0058559998869895935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,127,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,127,0.006346666564544042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,255,0.012719999998807907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,255,0.007567999884486198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,511,0.013013333082199097
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,131071,0.03817066550254822
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,511,0.007925333455204964
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,131071,0.14602667093276978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,1023,0.019071999937295914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,1023,0.009178666397929192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,2047,0.0286613330245018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,2047,0.01180800050497055
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,4095,0.041375999649365745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,4095,0.01821333294113477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,131071,0.007786666974425316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,8191,0.05991466840108236
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,8191,0.02367466688156128
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,131071,0.09221333265304565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,16383,0.09108799695968628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,16383,0.04254400233427683
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,32767,0.1312373379866282
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,1,0.009008000294367472
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,1,0.0058133335163195925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,32767,0.06508266429106395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,3,0.008725333337982496
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,3,0.005594666426380475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,7,0.008581333483258883
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,7,0.006010666489601135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,15,0.008874666566650072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,15,0.005578666925430298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,31,0.009370666618148485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,31,0.006058666855096817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,63,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,63,0.00578666664659977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,127,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,127,0.007194666812817256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,65535,0.11070400476455688
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,255,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,65535,0.22090667486190796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,255,0.00744000015159448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,511,0.014917333920796713
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,511,0.007920000081261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,1023,0.016085332880417507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,1023,0.008181333541870117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,2047,0.016293333222468693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,2047,0.009045333291093508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,4095,0.016122666498025257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,4095,0.010602666685978571
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,8191,0.018266666680574417
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,8191,0.012426666915416718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,16383,0.019999999552965164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,16383,0.014767999450365702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,32767,0.024085332949956257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,131071,0.2038080096244812
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,32767,0.0185759998857975
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,1,0.008629333227872849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,131071,0.4129706621170044
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,1,0.005781333272655805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,3,0.008602666358153025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,65535,0.02898666759332021
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,3,0.005744000275929769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,7,0.008538666491707167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,65535,0.02317333221435547
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,15,0.005834666391213735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,7,0.005648000165820122
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,15,0.008805333326260248
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,31,0.009488000224033991
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,31,0.00590933362642924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,63,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,63,0.005664000287652016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,127,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,127,0.006101333225766818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,255,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,255,0.007567999884486198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,511,0.015050667027632395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,511,0.007647999872763951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,131071,0.03953066716591517
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,2047,0.00897066667675972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,1023,0.016229332735141117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,131071,0.03054933249950409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,1023,0.008176000167926153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,2047,0.016538667182127636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,4095,0.01570133368174235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,4095,0.01027199998497963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,8191,0.01874133323629697
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,8191,0.012527999778588613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,16383,0.02040533348917961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,16383,0.014576000471909841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,32767,0.025045332809289295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,32767,0.01868266612291336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,1,0.010330666477481524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,1,0.005989333614706993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,3,0.010474666953086853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,3,0.005957333371043205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,7,0.010245333115259806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,65535,0.031130666534105938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,15,0.010245333115259806
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,7,0.005941333249211311
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,65535,0.02329600105683009
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,15,0.006095999851822853
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,31,0.01009599988659223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,31,0.005984000240763028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,63,0.010149333626031876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,63,0.005781333272655805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,127,0.010197333370645842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,127,0.006074666976928711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,255,0.013365333278973898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,255,0.007743999982873599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,511,0.019130667050679524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,511,0.008752000207702318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,1023,0.02794133375088374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,131071,0.03350399931271871
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,131071,0.06225599845250448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,1023,0.013658666362365087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,2047,0.043493335445721946
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,2047,0.016864000509182613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,4095,0.06922666728496552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,4095,0.02608533451954524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,8191,0.08910399675369263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,8191,0.040906667709350586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,16383,0.1297546625137329
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,16383,0.06463466584682465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,1,0.00847999999920527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,3,0.008474666625261307
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,1,0.005632000043988228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,3,0.005573333551486333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,7,0.008559999987483025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,32767,0.11192533373832703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,32767,0.217466672261556
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,7,0.005712000032265981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,15,0.008832000195980072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,15,0.0058666666348775225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,31,0.009455999980370203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,63,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,31,0.0058133335163195925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,63,0.0058559998869895935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,127,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,127,0.006069333602984746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,255,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,255,0.007573333258430163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,511,0.015226667126019796
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,511,0.00785600021481514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,1023,0.016229332735141117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,1023,0.00843733362853527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,65535,0.3955306609471639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,65535,0.20778665939966837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,2047,0.016149333367745083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,2047,0.009189333145817121
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,4095,0.017050666113694508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,4095,0.010442666709423065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,8191,0.020879998803138733
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,8191,0.01249066616098086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,16383,0.023152001202106476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,16383,0.01479999969402949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,32767,0.037231999138991036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,32767,0.021002667645613354
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,1,0.010453333457310995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,65535,0.05342933535575867
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,1,0.006805333619316419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,65535,0.025994665920734406
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,3,0.006170666466156642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,3,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,7,0.010570666442314783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,7,0.006159999718268712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,15,0.010709332923094431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,15,0.005914666379491488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,31,0.010480000327030817
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,31,0.006069333602984746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,63,0.01051733394463857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,63,0.006042666733264923
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,127,0.010661333799362183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,127,0.006298666819930077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,255,0.02075200031201045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,131071,0.08542399605115254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,131071,0.04308266441027323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,511,0.013232000172138214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,255,0.008538666491707167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,511,0.028160000840822857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,1023,0.047322665651639305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,131071,0.7513546943664551
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,1023,0.019280000279347103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,2047,0.06675733129183452
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,2047,0.025472000241279602
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,131071,0.398144006729126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,4095,0.03955200066169103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,4095,0.08691199620564778
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,8191,0.1285920043786367
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,8191,0.06341333190600078
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,16383,0.219376007715861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,1,0.015322666615247726
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,16383,0.11214933792750041
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,1,0.006560000280539195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,3,0.01552533358335495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,3,0.006528000036875407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,7,0.015370666980743408
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,7,0.006549333532651265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,15,0.01540800059835116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,15,0.006634666894872983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,31,0.015647999942302704
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,31,0.006501333167155583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,63,0.015146666516860327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,63,0.006528000036875407
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,127,0.01543466622630755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,127,0.007002666592597961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,32767,0.40059733390808105
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,32767,0.20683199167251587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,255,0.030458666384220123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,255,0.013157332936922709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,511,0.046709333856900535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,511,0.019343999524911244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,1023,0.06276800235112508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,1023,0.025013332565625507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,2047,0.08688533306121826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,2047,0.03978666663169861
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,4095,0.12981866796811423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,4095,0.0635040005048116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,8191,0.11121066411336263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,1,0.025040000677108765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,1,0.009679999823371569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,8191,0.2208426594734192
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,3,0.02508266766866048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,3,0.009701333319147428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,7,0.02481066683928172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,7,0.009893333539366722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,65535,0.39614399274190265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,15,0.025066666305065155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,15,0.009770666559537252
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,31,0.024842667082945507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,16383,0.4049919843673706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,31,0.009872000043590864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,16383,0.20546134312947592
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,63,0.02476799984773
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,63,0.009701333319147428
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,65535,0.7607306639353434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,127,0.02479466547568639
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,127,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,255,0.0506879985332489
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,255,0.01932799940307935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,511,0.06530133386452992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,511,0.025397333006064098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,1023,0.04027199993530909
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,1023,0.08410666386286418
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,2047,0.06458666423956554
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,1,0.008538666491707167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,1,0.005690666536490123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,2047,0.13166399796803793
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,3,0.008559999987483025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,3,0.00589866687854131
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,7,0.008463999877373377
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,7,0.006319999694824219
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,15,0.009072000160813332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,4095,0.22426666816075644
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,15,0.0058399997651577
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,31,0.009541333342591921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,4095,0.11109333237012227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,31,0.006111999973654747
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,63,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,127,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,127,0.006351999938488007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,63,0.0057920000205437345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,255,0.007530666887760162
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,255,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,511,0.015487999965747198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,511,0.008005333443482717
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,1023,0.013466666142145792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,1023,0.008165333420038223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,8191,0.4149706761042277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,2047,0.01639466608564059
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,2047,0.009301333377758661
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,4095,0.017887999614079792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,4095,0.010570666442314783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,8191,0.027077332139015198
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,8191,0.20638400316238403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,8191,0.01404800017674764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,16383,0.03262399882078171
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,16383,0.016666666915019352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,32767,0.05505066613356272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,32767,0.023941333095232647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,65535,0.049141332507133484
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,1,0.04399999976158142
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,65535,0.08105599880218506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,1,0.013983999689420065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,3,0.043696001172065735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,3,0.013888000200192133
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,7,0.044112001856168113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,7,0.013904000322024027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,15,0.04368533194065094
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,15,0.014090667168299357
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,31,0.044106667240460716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,31,0.01379199946920077
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,63,0.044266665975252785
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,131071,0.1377173364162445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,131071,0.06739733119805653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,63,0.013552000125249227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,127,0.04507733384768168
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,127,0.015743999431530636
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,255,0.05397333204746246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,255,0.021946666141351063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,511,0.037503999968369804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,511,0.07399466633796692
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,1023,0.11869866649309795
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,1023,0.06115733087062836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,2047,0.21592533588409424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,2047,0.10785067081451416
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,1,0.08002666632334392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,1,0.023599999646345775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,3,0.023605334262053173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,4095,0.4002240101496379
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,4095,0.20401066541671753
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,7,0.08049599826335907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,7,0.02327466756105423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,15,0.023621333142121632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,15,0.07979733248551686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,31,0.08036800225575765
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,31,0.02348266790310542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,63,0.08025600016117096
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,63,0.02349333216746648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,127,0.028778667251269024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,127,0.08321066697438557
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,255,0.09669333696365356
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,255,0.04188799858093262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,1,0.008954666554927826
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,1,0.005930666501323382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,3,0.008986666798591614
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,511,0.1325759987036387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,511,0.06471466521422069
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,3,0.005957333371043205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,7,0.008634666601816813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,7,0.006053333481152852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,15,0.008896000062425932
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,15,0.0058133335163195925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,31,0.009786666681369146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,31,0.005722666780153911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,1023,0.11010133226712544
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,63,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,63,0.006021333237489064
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,1023,0.21631999810536703
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,127,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,127,0.006213333457708359
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,255,0.011541333049535751
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,255,0.007776000226537387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,511,0.012794667234023413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,511,0.007802666475375493
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,1023,0.013258667041858038
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,1023,0.008623999853928884
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,2047,0.40487468242645264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,2047,0.021525333325068157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,2047,0.010213333492477735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,4095,0.024826665719350178
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,4095,0.011802667131026586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,2047,0.2025173306465149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,8191,0.03679466744263967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,8191,0.015834666788578033
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,16383,0.056186666091283165
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,16383,0.02734400083621343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,32767,0.08453333377838135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,32767,0.04046933352947235
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,65535,0.13742933670679727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,65535,0.06963733335336049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,3,0.08014933268229167
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,131071,0.23341333866119385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,131071,0.11414399743080139
