framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,beam_width,attn_dtype,kv_cache_dtype,step,latency
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,96,1,128,1,fp8,fp8,0,70.86961364746094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,96,2,128,1,fp8,fp8,0,72.20236206054688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,96,4,128,1,fp8,fp8,0,72.73207600911458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,96,8,128,1,fp8,fp8,0,73.00917053222656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,96,128,1,float16,float16,0,50.43836466471354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,96,128,1,float16,fp8,0,50.18498738606771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,1,128,1,float16,fp8,0,51.34600321451823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,1,128,1,float16,float16,0,50.02139790852865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,1,128,1,fp8,fp8,0,33.16233571370443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,2,128,1,float16,float16,0,50.77131652832031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,2,128,1,float16,fp8,0,50.62666320800781
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,2,128,1,fp8,fp8,0,32.873514811197914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,4,128,1,float16,float16,0,50.49102274576823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,4,128,1,float16,fp8,0,50.58826192220052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,4,128,1,fp8,fp8,0,33.96392059326172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,8,128,1,float16,fp8,0,50.2066650390625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,8,128,1,float16,float16,0,50.78373718261719
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,96,8,128,1,fp8,fp8,0,33.082628885904946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,1,128,1,float16,float16,0,25.97039031982422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,1,128,1,fp8,fp8,0,16.70500310262044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,1,128,1,float16,fp8,0,25.43591562906901
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,96,128,1,float16,fp8,0,25.273585001627605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,2,128,1,float16,float16,0,25.152422587076824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,2,128,1,float16,fp8,0,25.883806864420574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,96,128,1,fp8,fp8,0,17.208730061848957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,96,128,1,float16,float16,0,25.469589233398438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,2,128,1,fp8,fp8,0,16.78109359741211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,4,128,1,float16,float16,0,25.651285807291668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,4,128,1,float16,fp8,0,25.253679911295574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,4,128,1,fp8,fp8,0,16.6953125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,8,128,1,float16,float16,0,25.916976928710938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,8,128,1,float16,fp8,0,25.3864262898763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,96,8,128,1,fp8,fp8,0,16.900352478027344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,1,128,1,float16,float16,0,12.882080078125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,96,128,1,float16,float16,0,12.99447504679362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,1,128,1,float16,fp8,0,13.312330881754557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,1,128,1,fp8,fp8,0,8.609493255615234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,96,128,1,float16,fp8,0,12.992197672526041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,96,128,1,fp8,fp8,0,8.718063990275065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,2,128,1,fp8,fp8,0,8.651552200317383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,2,128,1,float16,fp8,0,12.807540893554688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,4,128,1,fp8,fp8,0,8.761541366577148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,4,128,1,float16,float16,0,13.319386800130209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,4,128,1,float16,fp8,0,12.998891194661459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,2,128,1,float16,float16,0,13.212165832519531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,8,128,1,float16,float16,0,12.988698323567709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,8,128,1,float16,fp8,0,12.993695576985678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,96,8,128,1,fp8,fp8,0,8.631381352742514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,1,128,1,float16,float16,0,62.2879892985026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,1,128,1,fp8,fp8,0,38.567179361979164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,1,128,1,float16,fp8,0,62.707295735677086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,2,128,1,fp8,fp8,0,38.72986602783203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,2,128,1,float16,fp8,0,60.304056803385414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,2,128,1,float16,float16,0,63.44928995768229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,4,128,1,float16,float16,0,60.0855712890625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,4,128,1,fp8,fp8,0,38.847241719563804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,4,128,1,float16,fp8,0,62.24116516113281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,8,128,1,float16,float16,0,64.01418050130208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,8,128,1,fp8,fp8,0,38.71611785888672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,96,8,128,1,float16,fp8,0,62.1718495686849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,1,128,1,float16,float16,0,29.326065063476562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,1,128,1,float16,fp8,0,29.60119883219401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,96,128,1,float16,float16,0,29.498934427897137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,96,128,1,float16,fp8,0,29.01092783610026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,1,128,1,fp8,fp8,0,19.156949361165363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,2,128,1,float16,float16,0,29.157936096191406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,2,128,1,float16,fp8,0,29.2976811726888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,2,128,1,fp8,fp8,0,20.069039662679035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,4,128,1,float16,float16,0,29.278432210286457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,96,128,1,fp8,fp8,0,19.68567403157552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,4,128,1,fp8,fp8,0,19.265445709228516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,4,128,1,float16,fp8,0,28.752090454101562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,8,128,1,float16,float16,0,29.292516072591145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,8,128,1,float16,fp8,0,29.488026936848957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,1,128,1,float16,float16,0,14.662245432535807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,1,128,1,float16,fp8,0,14.755060831705729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,96,8,128,1,fp8,fp8,0,20.126847585042317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,1,128,1,fp8,fp8,0,9.929712295532227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,96,128,1,float16,float16,0,14.979583740234375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,2,128,1,float16,float16,0,14.591466267903646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,96,128,1,float16,fp8,0,14.782703399658203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,96,128,1,fp8,fp8,0,10.296709060668945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,2,128,1,fp8,fp8,0,10.15339215596517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,2,128,1,float16,fp8,0,14.691824595133463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,4,128,1,float16,float16,0,14.802261352539062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,4,128,1,float16,fp8,0,14.710992177327475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,4,128,1,fp8,fp8,0,9.771205266316732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,8,128,1,float16,float16,0,14.766309102376303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,8,128,1,fp8,fp8,0,10.279674530029297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,96,8,128,1,float16,fp8,0,14.612276713053385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,1,128,1,float16,float16,0,7.502288182576497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,96,128,1,float16,float16,0,7.402746836344401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,1,128,1,fp8,fp8,0,5.067610740661621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,1,128,1,float16,fp8,0,7.526069641113281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,2,128,1,float16,float16,0,7.384623845418294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,96,128,1,fp8,fp8,0,5.145717302958171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,2,128,1,fp8,fp8,0,5.097994804382324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,96,128,1,float16,fp8,0,7.452047983805339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,2,128,1,float16,fp8,0,7.3932749430338545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,4,128,1,float16,float16,0,7.6952158610026045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,4,128,1,float16,fp8,0,7.466970443725586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,4,128,1,fp8,fp8,0,5.087375958760579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,8,128,1,float16,float16,0,7.51370112101237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,8,128,1,fp8,fp8,0,5.099210739135742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,96,8,128,1,float16,fp8,0,7.427109400431315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,1,128,1,float16,float16,0,40.981300354003906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,1,128,1,fp8,fp8,0,27.65685780843099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,1,128,1,float16,fp8,0,41.9647471110026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,2,128,1,float16,float16,0,41.11176554361979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,2,128,1,fp8,fp8,0,27.849461873372395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,2,128,1,float16,fp8,0,41.261680603027344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,4,128,1,float16,float16,0,41.53283182779948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,4,128,1,fp8,fp8,0,27.669174194335938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,4,128,1,float16,fp8,0,40.656463623046875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,8,128,1,float16,float16,0,42.262794494628906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,8,128,1,fp8,fp8,0,27.99139658610026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,96,8,128,1,float16,fp8,0,40.94923655192057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,96,128,1,float16,float16,0,21.02563730875651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,96,128,1,float16,fp8,0,20.654986063639324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,1,128,1,float16,float16,0,20.409168243408203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,1,128,1,float16,fp8,0,21.11277898152669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,1,128,1,fp8,fp8,0,13.915322621663412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,2,128,1,float16,float16,0,20.39612325032552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,2,128,1,float16,fp8,0,20.772688547770183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,2,128,1,fp8,fp8,0,13.914101918538412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,4,128,1,float16,float16,0,20.380565643310547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,4,128,1,float16,fp8,0,20.874715169270832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,4,128,1,fp8,fp8,0,14.01534907023112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,96,128,1,fp8,fp8,0,14.399487813313803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,8,128,1,float16,float16,0,20.75212224324544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,8,128,1,float16,fp8,0,20.53343963623047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,96,8,128,1,fp8,fp8,0,14.014155069986979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,1,128,1,float16,float16,0,10.62994130452474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,1,128,1,fp8,fp8,0,7.108938852945964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,1,128,1,float16,fp8,0,10.453392028808594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,2,128,1,float16,float16,0,10.423519770304361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,96,128,1,float16,fp8,0,10.774218241373697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,96,128,1,float16,float16,0,10.44707171122233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,2,128,1,float16,fp8,0,10.363904317220053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,96,128,1,fp8,fp8,0,7.402293523152669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,2,128,1,fp8,fp8,0,7.0324052174886065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,4,128,1,float16,float16,0,10.476746877034506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,4,128,1,float16,fp8,0,10.413338979085287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,4,128,1,fp8,fp8,0,7.29416020711263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,8,128,1,fp8,fp8,0,7.1183624267578125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,8,128,1,float16,fp8,0,10.32812245686849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,96,8,128,1,float16,float16,0,10.395242691040039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,96,128,1,float16,float16,0,5.207338651021321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,1,128,1,float16,float16,0,5.388069152832031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,1,128,1,float16,fp8,0,5.3364213307698565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,1,128,1,fp8,fp8,0,3.7016159693400064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,96,128,1,float16,fp8,0,5.37339719136556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,2,128,1,float16,float16,0,5.261173248291016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,2,128,1,float16,fp8,0,5.419151941935222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,2,128,1,fp8,fp8,0,3.709994633992513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,4,128,1,float16,float16,0,5.185791969299316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,96,128,1,fp8,fp8,0,3.7868639628092446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,4,128,1,fp8,fp8,0,3.7095839182535806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,4,128,1,float16,fp8,0,5.372549057006836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,8,128,1,float16,float16,0,5.355248133341472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,8,128,1,float16,fp8,0,5.3849226633707685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,96,8,128,1,fp8,fp8,0,3.730799992879232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,96,1,128,1,fp8,fp8,0,36.906410217285156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,96,2,128,1,fp8,fp8,0,37.196207682291664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,96,4,128,1,fp8,fp8,0,37.61162567138672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,96,8,128,1,fp8,fp8,0,37.35067240397135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,1,128,1,float16,float16,0,27.15233612060547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,96,128,1,float16,float16,0,27.685829162597656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,1,128,1,float16,fp8,0,27.27484893798828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,96,128,1,float16,fp8,0,27.237279256184895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,1,128,1,fp8,fp8,0,18.30296579996745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,2,128,1,float16,float16,0,27.305684407552082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,2,128,1,float16,fp8,0,27.070688883463543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,2,128,1,fp8,fp8,0,18.59875233968099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,4,128,1,float16,float16,0,27.12017059326172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,4,128,1,float16,fp8,0,27.181248982747395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,8,128,1,float16,float16,0,27.41627248128255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,4,128,1,fp8,fp8,0,18.714773813883465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,8,128,1,fp8,fp8,0,18.49173863728841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,96,8,128,1,float16,fp8,0,27.098533630371094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,1,128,1,float16,float16,0,13.880821228027344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,1,128,1,float16,fp8,0,13.654106140136719
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,1,128,1,fp8,fp8,0,9.302474975585938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,2,128,1,float16,float16,0,13.680063883463541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,96,128,1,float16,float16,0,13.804512023925781
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,2,128,1,fp8,fp8,0,9.31446393330892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,96,128,1,float16,fp8,0,13.869023640950521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,2,128,1,float16,fp8,0,13.841498057047525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,96,128,1,fp8,fp8,0,9.815530776977539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,4,128,1,float16,float16,0,13.558666229248047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,4,128,1,float16,fp8,0,13.557236989339193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,4,128,1,fp8,fp8,0,9.329504013061523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,8,128,1,fp8,fp8,0,9.63478914896647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,8,128,1,float16,float16,0,13.720864613850912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,96,8,128,1,float16,fp8,0,13.562394460042318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,1,128,1,float16,float16,0,6.919685363769531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,96,128,1,float16,float16,0,7.131594975789388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,1,128,1,float16,fp8,0,6.999514897664388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,1,128,1,fp8,fp8,0,4.719386736551921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,2,128,1,float16,float16,0,6.915413538614909
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,2,128,1,float16,fp8,0,6.804186503092448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,96,128,1,float16,fp8,0,6.809749603271484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,96,128,1,fp8,fp8,0,4.93173344930013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,2,128,1,fp8,fp8,0,4.723631858825684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,4,128,1,fp8,fp8,0,4.756890614827474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,4,128,1,float16,fp8,0,6.814490636189778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,4,128,1,float16,float16,0,6.963178634643555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,8,128,1,float16,float16,0,6.9454294840494795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,8,128,1,fp8,fp8,0,4.749919891357422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,96,8,128,1,float16,fp8,0,6.793573379516602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,1,128,1,float16,float16,0,3.5939146677652993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,1,128,1,float16,fp8,0,3.42195192972819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,1,128,1,fp8,fp8,0,2.562016010284424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,2,128,1,float16,float16,0,3.474026679992676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,96,128,1,float16,float16,0,3.498799959818522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,2,128,1,fp8,fp8,0,2.5592212677001953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,2,128,1,float16,fp8,0,3.548661231994629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,96,128,1,float16,fp8,0,3.477183977762858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,4,128,1,float16,fp8,0,3.4647413889567056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,96,128,1,fp8,fp8,0,2.631295998891195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,4,128,1,float16,float16,0,3.528794606526693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,4,128,1,fp8,fp8,0,2.5631945927937827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,8,128,1,float16,float16,0,3.486858685811361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,8,128,1,float16,fp8,0,3.5360053380330405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,96,8,128,1,fp8,fp8,0,2.564799944559733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,96,1,128,1,fp8,fp8,0,22.196177164713543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,96,2,128,1,float16,float16,0,32.17180887858073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,96,2,128,1,fp8,fp8,0,23.016766866048176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,96,4,128,1,fp8,fp8,0,22.349594116210938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,96,8,128,1,float16,float16,0,32.94421895345052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,1,128,1,float16,float16,0,15.963520050048828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,96,8,128,1,fp8,fp8,0,22.9901606241862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,1,128,1,fp8,fp8,0,11.476202646891275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,1,128,1,float16,fp8,0,15.90384038289388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,96,128,1,float16,float16,0,16.4443359375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,96,128,1,fp8,fp8,0,11.749268849690756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,96,128,1,float16,fp8,0,16.389066060384113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,2,128,1,float16,float16,0,16.09169642130534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,2,128,1,float16,fp8,0,16.283925374348957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,2,128,1,fp8,fp8,0,11.150890350341797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,4,128,1,float16,fp8,0,15.74334462483724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,4,128,1,float16,float16,0,16.049989064534504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,4,128,1,fp8,fp8,0,11.209125518798828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,8,128,1,float16,float16,0,16.119274139404297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,1,128,1,float16,float16,0,8.105791727701822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,8,128,1,float16,fp8,0,16.243642171223957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,1,128,1,float16,fp8,0,8.043477376302084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,96,8,128,1,fp8,fp8,0,11.219183603922525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,1,128,1,fp8,fp8,0,5.579349517822266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,96,128,1,float16,fp8,0,8.136042912801107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,96,128,1,fp8,fp8,0,5.912000020345052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,2,128,1,float16,float16,0,8.070037206013998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,2,128,1,float16,fp8,0,8.005013147989908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,2,128,1,fp8,fp8,0,5.584496180216472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,96,128,1,float16,float16,0,8.293242772420248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,4,128,1,float16,float16,0,8.021615982055664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,4,128,1,float16,fp8,0,7.716400146484375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,4,128,1,fp8,fp8,0,5.593343734741211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,8,128,1,float16,float16,0,8.17854372660319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,8,128,1,fp8,fp8,0,5.612080256144206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,96,8,128,1,float16,fp8,0,8.029354731241861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,1,128,1,float16,float16,0,3.937328020731608
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,96,128,1,float16,float16,0,4.026837348937988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,1,128,1,float16,fp8,0,4.055530548095703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,96,128,1,float16,fp8,0,4.126447995503743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,1,128,1,fp8,fp8,0,2.931898752848307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,2,128,1,float16,float16,0,3.9528799057006836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,2,128,1,float16,fp8,0,3.9985386530558267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,2,128,1,fp8,fp8,0,2.930762608846029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,96,128,1,fp8,fp8,0,3.0748427708943686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,4,128,1,float16,float16,0,4.027632077534993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,4,128,1,float16,fp8,0,3.905791918436686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,4,128,1,fp8,fp8,0,2.939743995666504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,8,128,1,float16,float16,0,3.963306744893392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,8,128,1,float16,fp8,0,3.8883838653564453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,1,128,1,float16,float16,0,2.1276426315307617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,1,128,1,float16,fp8,0,2.0925866762797036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,96,8,128,1,fp8,fp8,0,2.9437599182128906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,96,128,1,float16,float16,0,2.14301331837972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,1,128,1,fp8,fp8,0,1.6169439951578777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,96,128,1,float16,fp8,0,2.163157304128011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,2,128,1,float16,float16,0,2.1292746861775718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,96,128,1,fp8,fp8,0,1.6789387067159016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,2,128,1,float16,fp8,0,2.0975252787272134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,2,128,1,fp8,fp8,0,1.6177813212076824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,4,128,1,float16,float16,0,2.128981272379557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,4,128,1,fp8,fp8,0,1.6184213956197102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,4,128,1,float16,fp8,0,2.092378616333008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,8,128,1,float16,float16,0,2.1325653394063315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,8,128,1,float16,fp8,0,2.10208527247111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,96,8,128,1,fp8,fp8,0,1.626047929128011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,96,1,128,1,fp8,fp8,0,22.987904866536457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,96,2,128,1,fp8,fp8,0,22.55669403076172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,96,4,128,1,fp8,fp8,0,22.604080200195312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,96,8,128,1,fp8,fp8,0,22.727887471516926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,1,128,1,float16,fp8,0,15.489578247070312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,1,128,1,float16,float16,0,16.09289042154948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,96,128,1,float16,float16,0,16.349594116210938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,96,128,1,float16,fp8,0,16.12704594930013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,1,128,1,fp8,fp8,0,11.242933909098307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,2,128,1,float16,fp8,0,15.447125752766928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,2,128,1,float16,float16,0,15.947786966959635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,2,128,1,fp8,fp8,0,11.297823588053385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,4,128,1,float16,float16,0,15.715231577555338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,4,128,1,float16,fp8,0,15.455343882242838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,4,128,1,fp8,fp8,0,11.50323232014974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,8,128,1,float16,fp8,0,15.546554565429688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,8,128,1,float16,float16,0,15.418538411458334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,96,8,128,1,fp8,fp8,0,11.257471720377604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,1,128,1,float16,fp8,0,7.618597030639648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,1,128,1,float16,float16,0,7.792405446370442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,2,128,1,float16,float16,0,7.5846506754557295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,1,128,1,fp8,fp8,0,5.629632314046224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,2,128,1,float16,fp8,0,7.5759627024332685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,96,128,1,float16,fp8,0,8.072944005330404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,2,128,1,fp8,fp8,0,5.631594975789388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,96,128,1,fp8,fp8,0,6.198618570963542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,96,128,1,float16,float16,0,8.084304173787435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,4,128,1,float16,float16,0,7.627322514851888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,4,128,1,float16,fp8,0,7.6507415771484375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,4,128,1,fp8,fp8,0,5.656970977783203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,8,128,1,float16,float16,0,7.80617078145345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,8,128,1,float16,fp8,0,7.7393544514973955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,96,8,128,1,fp8,fp8,0,5.700858434041341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,1,128,1,float16,float16,0,3.7823893229166665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,1,128,1,float16,fp8,0,3.708906809488932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,1,128,1,fp8,fp8,0,2.904111862182617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,96,128,1,float16,float16,0,4.029018719991048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,2,128,1,float16,float16,0,3.840789477030436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,2,128,1,fp8,fp8,0,2.9071785608927407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,2,128,1,float16,fp8,0,3.716800053914388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,96,128,1,float16,fp8,0,3.8805761337280273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,96,128,1,fp8,fp8,0,3.1371946334838867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,4,128,1,float16,float16,0,3.783733367919922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,4,128,1,float16,fp8,0,3.7955732345581055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,4,128,1,fp8,fp8,0,2.9121599197387695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,8,128,1,float16,float16,0,3.8467839558919272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,8,128,1,float16,fp8,0,3.7307891845703125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,96,8,128,1,fp8,fp8,0,2.923818588256836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,1,128,1,float16,float16,0,2.0172266960144043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,1,128,1,float16,fp8,0,1.9504052797953289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,96,128,1,float16,float16,0,2.0488640467325845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,1,128,1,fp8,fp8,0,1.5513653755187988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,2,128,1,float16,float16,0,1.9931893348693848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,96,128,1,float16,fp8,0,2.026026725769043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,2,128,1,float16,fp8,0,1.978432019551595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,2,128,1,fp8,fp8,0,1.5515732765197754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,96,128,1,fp8,fp8,0,1.658357302347819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,4,128,1,float16,float16,0,1.9951732953389485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,4,128,1,float16,fp8,0,1.9592426617940266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,4,128,1,fp8,fp8,0,1.553285280863444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,8,128,1,float16,float16,0,1.9981706937154133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,8,128,1,float16,fp8,0,1.9855626424153645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,1,128,1,float16,float16,0,1.1053386529286702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,96,8,128,1,fp8,fp8,0,1.5582985877990723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,96,128,1,float16,float16,0,1.1276373068491619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,1,128,1,float16,fp8,0,1.0792373021443684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,96,128,1,fp8,fp8,0,0.9285706679026285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,96,128,1,float16,fp8,0,1.10315736134847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,1,128,1,fp8,fp8,0,0.8799253304799398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,2,128,1,float16,float16,0,1.1049813429514568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,2,128,1,float16,fp8,0,1.0817173322041829
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,2,128,1,fp8,fp8,0,0.8807840347290039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,4,128,1,float16,float16,0,1.107199986775716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,4,128,1,float16,fp8,0,1.0840160051981609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,4,128,1,fp8,fp8,0,0.8823146820068359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,8,128,1,float16,float16,0,1.1169546445210774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,8,128,1,float16,fp8,0,1.0824639797210693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,96,8,128,1,fp8,fp8,0,0.8852372964223226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,96,1,128,1,fp8,fp8,0,14.218624114990234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,96,1,128,1,float16,float16,0,18.998485565185547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,96,2,128,1,fp8,fp8,0,14.21655527750651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,96,4,128,1,fp8,fp8,0,14.298261006673178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,96,8,128,1,fp8,fp8,0,14.311280568440756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,96,128,1,float16,float16,0,10.069040298461914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,1,128,1,float16,float16,0,9.555327733357748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,96,128,1,float16,fp8,0,10.116160074869791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,96,128,1,fp8,fp8,0,7.735962549845378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,1,128,1,float16,fp8,0,9.308154424031576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,1,128,1,fp8,fp8,0,7.119882583618164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,2,128,1,float16,float16,0,9.400954564412435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,2,128,1,float16,fp8,0,9.40176010131836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,2,128,1,fp8,fp8,0,7.1697438557942705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,4,128,1,float16,float16,0,9.469402949015299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,4,128,1,float16,fp8,0,9.208085378011068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,4,128,1,fp8,fp8,0,7.143178939819336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,8,128,1,float16,float16,0,9.359338760375977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,8,128,1,fp8,fp8,0,7.17796262105306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,1,128,1,float16,float16,0,4.553397178649902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,96,8,128,1,float16,fp8,0,9.39571762084961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,1,128,1,float16,fp8,0,4.502319971720378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,96,128,1,float16,float16,0,5.105072021484375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,2,128,1,float16,float16,0,4.557141304016113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,1,128,1,fp8,fp8,0,3.600325266520182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,2,128,1,float16,fp8,0,4.562677383422852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,96,128,1,fp8,fp8,0,3.945733388264974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,96,128,1,float16,fp8,0,4.833605448404948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,2,128,1,fp8,fp8,0,3.6037279764811196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,4,128,1,float16,float16,0,4.636170705159505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,4,128,1,float16,fp8,0,4.598623911539714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,4,128,1,fp8,fp8,0,3.6182241439819336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,8,128,1,float16,float16,0,4.591759999593099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,8,128,1,float16,fp8,0,4.4794613520304365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,96,8,128,1,fp8,fp8,0,3.6258665720621743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,1,128,1,float16,float16,0,2.3587679862976074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,1,128,1,float16,fp8,0,2.306858698527018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,96,128,1,float16,float16,0,2.490821361541748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,1,128,1,fp8,fp8,0,1.8761119842529297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,2,128,1,float16,float16,0,2.3550987243652344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,2,128,1,float16,fp8,0,2.3061280250549316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,96,128,1,float16,fp8,0,2.5033119519551597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,2,128,1,fp8,fp8,0,1.8789973258972168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,4,128,1,float16,float16,0,2.361290613810221
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,96,128,1,fp8,fp8,0,2.0573280652364097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,4,128,1,fp8,fp8,0,1.8821172714233398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,4,128,1,float16,fp8,0,2.3096747398376465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,8,128,1,float16,float16,0,2.3700159390767417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,8,128,1,float16,fp8,0,2.3465493520100913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,96,8,128,1,fp8,fp8,0,1.8918506304423015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,1,128,1,float16,float16,0,1.2579306761423747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,1,128,1,float16,fp8,0,1.2285066445668538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,1,128,1,fp8,fp8,0,1.016592025756836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,96,128,1,float16,fp8,0,1.2837759653727214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,96,128,1,float16,float16,0,1.3076159954071045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,2,128,1,float16,float16,0,1.261840025583903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,2,128,1,float16,fp8,0,1.22980801264445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,96,128,1,fp8,fp8,0,1.09825603167216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,2,128,1,fp8,fp8,0,1.017621358235677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,4,128,1,float16,float16,0,1.2722453276316326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,4,128,1,float16,fp8,0,1.2303573290507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,4,128,1,fp8,fp8,0,1.0189599990844727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,8,128,1,float16,float16,0,1.2650240262349446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,1,128,1,float16,float16,0,0.7149120171864828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,8,128,1,fp8,fp8,0,1.0207839806874592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,96,8,128,1,float16,fp8,0,1.2370293140411377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,96,128,1,float16,float16,0,0.729973316192627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,96,128,1,float16,fp8,0,0.7234880129496256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,96,128,1,fp8,fp8,0,0.6266560157140096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,1,128,1,float16,fp8,0,0.6966506640116373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,1,128,1,fp8,fp8,0,0.5883573293685913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,2,128,1,float16,float16,0,0.715994675954183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,2,128,1,float16,fp8,0,0.6975093682607015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,2,128,1,fp8,fp8,0,0.590837319691976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,4,128,1,float16,float16,0,0.7176640033721924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,4,128,1,float16,fp8,0,0.6957440376281738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,4,128,1,fp8,fp8,0,0.5914293527603149
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,8,128,1,float16,float16,0,0.7159732977549235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,8,128,1,float16,fp8,0,0.6981013615926107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,96,8,128,1,fp8,fp8,0,0.5913813511530558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,96,1,128,1,fp8,fp8,0,15.723604838053385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,96,2,128,1,fp8,fp8,0,15.76271947224935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,96,4,128,1,fp8,fp8,0,15.808943430582682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,96,8,128,1,fp8,fp8,0,15.88485336303711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,96,128,1,float16,float16,0,10.659829457600912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,96,128,1,float16,fp8,0,10.300005594889322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,1,128,1,float16,float16,0,9.589754740397135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,1,128,1,float16,fp8,0,9.43618647257487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,2,128,1,float16,float16,0,9.638794581095377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,1,128,1,fp8,fp8,0,7.795877456665039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,2,128,1,fp8,fp8,0,7.806202570597331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,2,128,1,float16,fp8,0,9.32476806640625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,4,128,1,float16,float16,0,9.73574956258138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,4,128,1,float16,fp8,0,9.634991963704428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,4,128,1,fp8,fp8,0,7.824426651000977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,8,128,1,float16,float16,0,9.83239491780599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,8,128,1,float16,fp8,0,9.567482630411783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,96,8,128,1,fp8,fp8,0,7.871824264526367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,1,128,1,float16,float16,0,4.738944053649902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,1,128,1,float16,fp8,0,4.565168062845866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,1,128,1,fp8,fp8,0,3.888970692952474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,2,128,1,float16,fp8,0,4.574901262919108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,2,128,1,float16,float16,0,4.721184094746907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,96,128,1,float16,fp8,0,5.092015902201335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,2,128,1,fp8,fp8,0,3.898501396179199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,96,128,1,float16,float16,0,5.333221435546875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,4,128,1,float16,float16,0,4.718240102132161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,96,128,1,fp8,fp8,0,4.355130513509114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,4,128,1,float16,fp8,0,4.597237269083659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,8,128,1,fp8,fp8,0,3.9330078760782876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,8,128,1,float16,fp8,0,4.67137082417806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,8,128,1,float16,float16,0,4.732517242431641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,96,4,128,1,fp8,fp8,0,3.902565320332845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,96,128,1,float16,float16,0,2.6215893427530923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,1,128,1,float16,float16,0,2.4185706774393716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,1,128,1,float16,fp8,0,2.3334986368815103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,1,128,1,fp8,fp8,0,1.9979573885599773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,96,128,1,float16,fp8,0,2.583029270172119
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,2,128,1,float16,fp8,0,2.3391146659851074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,2,128,1,float16,float16,0,2.4033546447753906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,96,128,1,fp8,fp8,0,2.2378719647725425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,2,128,1,fp8,fp8,0,2.005743980407715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,4,128,1,float16,float16,0,2.4047786394755044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,4,128,1,float16,fp8,0,2.3482666015625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,4,128,1,fp8,fp8,0,2.0028212865193686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,8,128,1,float16,fp8,0,2.3491360346476235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,8,128,1,fp8,fp8,0,2.0129013061523438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,96,8,128,1,float16,float16,0,2.415712038675944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,1,128,1,float16,float16,0,1.2554079691569011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,96,128,1,float16,float16,0,1.3692906697591145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,96,128,1,float16,fp8,0,1.3228053251902263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,1,128,1,fp8,fp8,0,1.0541653633117676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,2,128,1,float16,float16,0,1.2584266662597656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,1,128,1,float16,fp8,0,1.2209760348002117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,2,128,1,float16,fp8,0,1.2240266799926758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,2,128,1,fp8,fp8,0,1.058517297108968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,96,128,1,fp8,fp8,0,1.1814026832580566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,4,128,1,float16,float16,0,1.259178638458252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,4,128,1,float16,fp8,0,1.2274080117543538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,4,128,1,fp8,fp8,0,1.058517297108968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,8,128,1,fp8,fp8,0,1.0661760171254475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,8,128,1,float16,float16,0,1.2609333197275798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,96,8,128,1,float16,fp8,0,1.2300906976064045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,1,128,1,float16,float16,0,0.6860693295796713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,96,128,1,float16,float16,0,0.7231146494547526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,96,128,1,float16,fp8,0,0.7055413722991943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,1,128,1,fp8,fp8,0,0.5823946793874105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,96,128,1,fp8,fp8,0,0.6396906773249308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,1,128,1,float16,fp8,0,0.6692426999409994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,2,128,1,float16,float16,0,0.6879306634267172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,2,128,1,float16,fp8,0,0.6683626969655355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,2,128,1,fp8,fp8,0,0.58297065893809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,4,128,1,fp8,fp8,0,0.5835093259811401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,8,128,1,float16,float16,0,0.6940906842549642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,4,128,1,float16,float16,0,0.687397321065267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,8,128,1,float16,fp8,0,0.669818639755249
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,4,128,1,float16,fp8,0,0.6678400039672852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,96,8,128,1,fp8,fp8,0,0.5868746836980184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,1,128,1,float16,float16,0,0.4021759827931722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,1,128,1,float16,fp8,0,0.3906613190968831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,1,128,1,fp8,fp8,0,0.3352266550064087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,96,128,1,float16,fp8,0,0.4063520034154256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,96,128,1,float16,float16,0,0.41629334290822345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,96,128,1,fp8,fp8,0,0.37438400586446124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,2,128,1,float16,fp8,0,0.39047467708587646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,2,128,1,float16,float16,0,0.40427732467651367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,2,128,1,fp8,fp8,0,0.3364266554514567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,4,128,1,float16,fp8,0,0.3911733229955037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,4,128,1,float16,float16,0,0.4039200146993001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,4,128,1,fp8,fp8,0,0.33583466211954754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,8,128,1,float16,float16,0,0.40674134095509845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,8,128,1,float16,fp8,0,0.3925173282623291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,96,8,128,1,fp8,fp8,0,0.3375413417816162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,96,2,128,1,fp8,fp8,0,10.496922810872396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,96,1,128,1,fp8,fp8,0,10.510874430338541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,96,4,128,1,fp8,fp8,0,10.52239990234375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,96,8,128,1,fp8,fp8,0,10.571023941040039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,96,128,1,float16,float16,0,6.905205408732097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,1,128,1,float16,fp8,0,5.905882517496745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,96,128,1,fp8,fp8,0,5.877413431803386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,96,128,1,float16,fp8,0,6.618042627970378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,1,128,1,float16,float16,0,6.08299191792806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,2,128,1,float16,float16,0,6.090223948160808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,2,128,1,float16,fp8,0,6.101477305094401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,1,128,1,fp8,fp8,0,5.1941172281901045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,2,128,1,fp8,fp8,0,5.206149419148763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,4,128,1,float16,float16,0,6.086240132649739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,4,128,1,float16,fp8,0,5.93394660949707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,4,128,1,fp8,fp8,0,5.223370552062988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,8,128,1,float16,float16,0,6.2851517995198565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,8,128,1,fp8,fp8,0,5.243328094482422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,96,8,128,1,float16,fp8,0,5.912858963012695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,1,128,1,float16,float16,0,3.0420961380004883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,96,128,1,float16,fp8,0,3.335455894470215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,1,128,1,float16,fp8,0,2.9593547185262046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,1,128,1,fp8,fp8,0,2.607850710550944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,2,128,1,float16,float16,0,3.042698542277018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,96,128,1,float16,float16,0,3.4248107274373374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,2,128,1,fp8,fp8,0,2.616074721018473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,2,128,1,float16,fp8,0,2.952533404032389
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,4,128,1,float16,float16,0,3.0646241505940757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,4,128,1,float16,fp8,0,2.9554131825764975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,96,128,1,fp8,fp8,0,2.9716691970825195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,4,128,1,fp8,fp8,0,2.618197282155355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,8,128,1,float16,float16,0,3.0583839416503906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,8,128,1,float16,fp8,0,2.966479937235514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,1,128,1,float16,float16,0,1.5648639996846516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,96,8,128,1,fp8,fp8,0,2.6367786725362143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,1,128,1,float16,fp8,0,1.5161066055297852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,1,128,1,fp8,fp8,0,1.3494079907735188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,96,128,1,float16,float16,0,1.7642292976379395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,2,128,1,float16,fp8,0,1.523205280303955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,2,128,1,float16,float16,0,1.5678826967875164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,96,128,1,fp8,fp8,0,1.5306399663289387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,96,128,1,float16,fp8,0,1.691109339396159
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,2,128,1,fp8,fp8,0,1.3507092793782551
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,4,128,1,float16,float16,0,1.5704213778177898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,4,128,1,float16,fp8,0,1.527781327565511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,8,128,1,float16,float16,0,1.5792214075724285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,4,128,1,fp8,fp8,0,1.3548426628112793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,8,128,1,float16,fp8,0,1.527488072713216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,96,8,128,1,fp8,fp8,0,1.363034725189209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,1,128,1,float16,float16,0,0.8265973726908366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,1,128,1,fp8,fp8,0,0.7195413112640381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,2,128,1,float16,float16,0,0.8290879726409912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,96,128,1,float16,fp8,0,0.8778346379597982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,96,128,1,float16,float16,0,0.9002133210500082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,1,128,1,float16,fp8,0,0.8023786544799805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,96,128,1,fp8,fp8,0,0.8158613046010336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,2,128,1,float16,fp8,0,0.8041386604309082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,2,128,1,fp8,fp8,0,0.7200160026550293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,4,128,1,float16,float16,0,0.8300267060597738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,4,128,1,float16,fp8,0,0.8059679667154948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,4,128,1,fp8,fp8,0,0.7221813201904297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,8,128,1,float16,float16,0,0.8336640199025472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,8,128,1,float16,fp8,0,0.8099359671274821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,1,128,1,float16,float16,0,0.45981331666310626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,96,8,128,1,fp8,fp8,0,0.7263999780019125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,1,128,1,fp8,fp8,0,0.40375999609629315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,96,128,1,fp8,fp8,0,0.4475466807683309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,96,128,1,float16,float16,0,0.4875040054321289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,96,128,1,float16,fp8,0,0.476800004641215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,2,128,1,float16,float16,0,0.4608159859975179
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,2,128,1,float16,fp8,0,0.4457813501358032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,1,128,1,float16,fp8,0,0.44396265347798664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,4,128,1,float16,float16,0,0.4619893232981364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,2,128,1,fp8,fp8,0,0.4033386707305908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,4,128,1,float16,fp8,0,0.4447946548461914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,4,128,1,fp8,fp8,0,0.4042346477508545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,8,128,1,float16,float16,0,0.4626133441925049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,8,128,1,float16,fp8,0,0.4488106568654378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,96,8,128,1,fp8,fp8,0,0.40936533610026044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,1,128,1,float16,float16,0,0.2768213351567586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,1,128,1,float16,fp8,0,0.26716800530751544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,96,128,1,float16,float16,0,0.2878506580988566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,96,128,1,float16,fp8,0,0.28313066562016803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,96,128,1,fp8,fp8,0,0.2624586621920268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,1,128,1,fp8,fp8,0,0.23653332392374674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,2,128,1,float16,float16,0,0.27825067440668744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,2,128,1,float16,fp8,0,0.2678613265355428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,4,128,1,float16,float16,0,0.27849066257476807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,2,128,1,fp8,fp8,0,0.23729600509007773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,4,128,1,fp8,fp8,0,0.23805866638819376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,4,128,1,float16,fp8,0,0.2691733241081238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,8,128,1,float16,float16,0,0.27896533409754437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,8,128,1,fp8,fp8,0,0.2388746738433838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,96,8,128,1,float16,fp8,0,0.2712586720784505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,96,1,128,1,fp8,fp8,0,10.274805068969727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,96,2,128,1,fp8,fp8,0,10.329002380371094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,96,4,128,1,fp8,fp8,0,10.51477305094401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,96,8,128,1,fp8,fp8,0,10.922213236490885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,1,128,1,float16,float16,0,5.869306564331055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,1,128,1,float16,fp8,0,5.987701416015625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,96,128,1,float16,float16,0,7.537237167358398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,96,128,1,float16,fp8,0,7.357845306396484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,1,128,1,fp8,fp8,0,5.121967951456706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,2,128,1,float16,float16,0,6.06828244527181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,2,128,1,float16,fp8,0,5.908154805501302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,2,128,1,fp8,fp8,0,5.154015858968099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,4,128,1,float16,float16,0,5.926560084025065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,4,128,1,float16,fp8,0,5.880165100097656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,8,128,1,float16,float16,0,6.0509599049886065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,8,128,1,float16,fp8,0,6.1369069417317705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,4,128,1,fp8,fp8,0,5.27183469136556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,96,8,128,1,fp8,fp8,0,5.465466817220052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,1,128,1,float16,float16,0,2.874410629272461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,1,128,1,fp8,fp8,0,2.5586133003234863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,2,128,1,float16,float16,0,2.9118932088216147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,1,128,1,float16,fp8,0,2.8767518997192383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,96,128,1,float16,float16,0,3.7581653594970703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,2,128,1,float16,fp8,0,2.9094025293986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,2,128,1,fp8,fp8,0,2.568181355794271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,4,128,1,float16,float16,0,2.922618548075358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,96,128,1,float16,fp8,0,3.6237014134724936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,96,128,1,fp8,fp8,0,3.1698080698649087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,4,128,1,float16,fp8,0,2.936767896016439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,4,128,1,fp8,fp8,0,2.616741339365641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,8,128,1,fp8,fp8,0,2.732192039489746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,8,128,1,float16,float16,0,3.0234880447387695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,1,128,1,float16,float16,0,1.4552747408548992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,96,8,128,1,float16,fp8,0,3.033184051513672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,96,128,1,float16,float16,0,1.8875679969787598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,1,128,1,float16,fp8,0,1.4527680079142253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,96,128,1,float16,fp8,0,1.835312048594157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,1,128,1,fp8,fp8,0,1.2723466555277507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,96,128,1,fp8,fp8,0,1.5792320569356282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,2,128,1,float16,float16,0,1.4643200238545735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,2,128,1,float16,fp8,0,1.4720373153686523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,2,128,1,fp8,fp8,0,1.2872906525929768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,4,128,1,float16,float16,0,1.471471945444743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,4,128,1,float16,fp8,0,1.4764800071716309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,4,128,1,fp8,fp8,0,1.3194879690806072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,8,128,1,float16,float16,0,1.4971359570821126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,8,128,1,float16,fp8,0,1.5072533289591472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,96,8,128,1,fp8,fp8,0,1.3802132606506348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,1,128,1,float16,float16,0,0.7426559925079346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,1,128,1,float16,fp8,0,0.743173360824585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,96,128,1,float16,float16,0,0.9519413312276205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,1,128,1,fp8,fp8,0,0.6464586655298868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,96,128,1,float16,fp8,0,0.9185919761657715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,2,128,1,float16,float16,0,0.7478613058725992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,2,128,1,float16,fp8,0,0.7495946884155273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,96,128,1,fp8,fp8,0,0.8048266569773356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,2,128,1,fp8,fp8,0,0.65502401192983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,4,128,1,float16,float16,0,0.7487093607584635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,4,128,1,float16,fp8,0,0.7541706562042236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,4,128,1,fp8,fp8,0,0.6594506502151489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,8,128,1,float16,float16,0,0.7541546821594238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,8,128,1,float16,fp8,0,0.7565653324127197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,96,8,128,1,fp8,fp8,0,0.6935520172119141
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,1,128,1,float16,float16,0,0.3893706798553467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,96,128,1,float16,float16,0,0.4904906749725342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,96,128,1,float16,fp8,0,0.4723466634750366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,1,128,1,fp8,fp8,0,0.33583466211954754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,2,128,1,float16,float16,0,0.3898346821467082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,96,128,1,fp8,fp8,0,0.4103999932607015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,1,128,1,float16,fp8,0,0.38701868057250977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,2,128,1,float16,fp8,0,0.39288532733917236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,2,128,1,fp8,fp8,0,0.34061865011850995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,4,128,1,float16,float16,0,0.38892801602681476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,4,128,1,float16,fp8,0,0.3913866678873698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,4,128,1,fp8,fp8,0,0.34069331487019855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,8,128,1,float16,float16,0,0.3937866687774658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,8,128,1,fp8,fp8,0,0.3559039831161499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,96,128,1,float16,float16,0,0.2600160042444865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,96,8,128,1,float16,fp8,0,0.3936053514480591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,1,128,1,float16,float16,0,0.2080693244934082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,1,128,1,fp8,fp8,0,0.17533334096272787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,1,128,1,float16,fp8,0,0.20814400911331177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,96,128,1,float16,fp8,0,0.2508479952812195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,96,128,1,fp8,fp8,0,0.22086399793624878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,2,128,1,float16,float16,0,0.2095200022061666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,2,128,1,float16,fp8,0,0.20925867557525635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,2,128,1,fp8,fp8,0,0.17611199617385864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,4,128,1,float16,float16,0,0.2111413280169169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,4,128,1,float16,fp8,0,0.20990399519602457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,4,128,1,fp8,fp8,0,0.17710399627685547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,8,128,1,float16,float16,0,0.21104532480239868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,8,128,1,fp8,fp8,0,0.17987199624379477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,96,8,128,1,float16,fp8,0,0.21188799540201822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,1,128,1,float16,float16,0,0.11267200112342834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,96,128,1,float16,fp8,0,0.1508693297704061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,96,128,1,float16,float16,0,0.15432000160217285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,1,128,1,float16,fp8,0,0.11344533165295918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,96,128,1,fp8,fp8,0,0.11733333269755046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,1,128,1,fp8,fp8,0,0.09660800298055013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,2,128,1,float16,float16,0,0.11355200409889221
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,2,128,1,fp8,fp8,0,0.0965280036131541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,2,128,1,float16,fp8,0,0.11348266402880351
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,4,128,1,float16,float16,0,0.11268799503644307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,8,128,1,float16,fp8,0,0.11412266890207927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,4,128,1,float16,fp8,0,0.1137600044409434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,4,128,1,fp8,fp8,0,0.09737066427866618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,8,128,1,float16,float16,0,0.11390399932861328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,96,8,128,1,fp8,fp8,0,0.10071466366449992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,96,1,128,1,fp8,fp8,0,8.559642791748047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,96,2,128,1,fp8,fp8,0,8.622709274291992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,96,4,128,1,fp8,fp8,0,8.752944310506185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,96,8,128,1,fp8,fp8,0,9.126768112182617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,96,128,1,float16,float16,0,6.274576187133789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,96,128,1,float16,fp8,0,6.01260248819987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,96,128,1,fp8,fp8,0,5.455909093221028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,1,128,1,float16,float16,0,4.580869356791179
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,1,128,1,float16,fp8,0,4.520885467529297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,1,128,1,fp8,fp8,0,4.2429758707682295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,2,128,1,float16,float16,0,4.6049652099609375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,2,128,1,float16,fp8,0,4.555040041605632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,4,128,1,float16,float16,0,4.6462507247924805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,2,128,1,fp8,fp8,0,4.269136110941569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,4,128,1,float16,fp8,0,4.596277236938477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,4,128,1,fp8,fp8,0,4.364282608032227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,8,128,1,float16,float16,0,4.759594599405925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,8,128,1,float16,fp8,0,4.788469314575195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,1,128,1,float16,float16,0,2.242784023284912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,1,128,1,float16,fp8,0,2.2285866737365723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,96,8,128,1,fp8,fp8,0,4.564842542012532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,96,128,1,float16,float16,0,3.141514778137207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,1,128,1,fp8,fp8,0,2.111637274424235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,2,128,1,float16,fp8,0,2.2642505963643393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,2,128,1,float16,float16,0,2.2628426551818848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,2,128,1,fp8,fp8,0,2.1233654022216797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,96,128,1,fp8,fp8,0,2.7173385620117188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,4,128,1,float16,float16,0,2.2857066790262857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,4,128,1,float16,fp8,0,2.2769813537597656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,4,128,1,fp8,fp8,0,2.1721760431925454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,8,128,1,float16,float16,0,2.3780159950256348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,96,128,1,float16,fp8,0,2.9905014038085938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,8,128,1,fp8,fp8,0,2.266122659047445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,96,8,128,1,float16,fp8,0,2.3714399337768555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,1,128,1,float16,float16,0,1.1299306551615398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,1,128,1,float16,fp8,0,1.1250080267588298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,96,128,1,float16,float16,0,1.5696907043457031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,96,128,1,float16,fp8,0,1.4976107279459636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,1,128,1,fp8,fp8,0,1.0527946949005127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,2,128,1,float16,float16,0,1.1390559673309326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,2,128,1,float16,fp8,0,1.147157351175944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,96,128,1,fp8,fp8,0,1.365455945332845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,4,128,1,float16,float16,0,1.1424586772918701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,2,128,1,fp8,fp8,0,1.073472023010254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,4,128,1,float16,fp8,0,1.145850658416748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,4,128,1,fp8,fp8,0,1.088362693786621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,8,128,1,float16,float16,0,1.180725336074829
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,8,128,1,float16,fp8,0,1.159605344136556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,96,8,128,1,fp8,fp8,0,1.1508959929148357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,1,128,1,float16,float16,0,0.5767146746317545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,1,128,1,float16,fp8,0,0.5785013437271118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,1,128,1,fp8,fp8,0,0.5317333141962687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,96,128,1,float16,fp8,0,0.757962703704834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,2,128,1,float16,float16,0,0.5813066562016805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,96,128,1,float16,float16,0,0.7979199886322021
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,96,128,1,fp8,fp8,0,0.6837226549784342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,2,128,1,float16,fp8,0,0.5811839898427328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,2,128,1,fp8,fp8,0,0.5414559841156006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,4,128,1,float16,float16,0,0.5850293238957723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,4,128,1,fp8,fp8,0,0.5441439946492513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,4,128,1,float16,fp8,0,0.5859839916229248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,8,128,1,float16,float16,0,0.5902346769968668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,8,128,1,float16,fp8,0,0.589850664138794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,96,8,128,1,fp8,fp8,0,0.5767840147018433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,1,128,1,float16,float16,0,0.3009493350982666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,1,128,1,float16,fp8,0,0.3017973303794861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,1,128,1,fp8,fp8,0,0.2770773371060689
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,2,128,1,float16,float16,0,0.3022293249766032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,96,128,1,float16,float16,0,0.41101332505544025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,96,128,1,float16,fp8,0,0.39363733927408856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,96,128,1,fp8,fp8,0,0.3514719804128011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,2,128,1,float16,fp8,0,0.30426132678985596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,2,128,1,fp8,fp8,0,0.28123732407887775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,4,128,1,float16,float16,0,0.30300267537434894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,4,128,1,float16,fp8,0,0.3049546678860982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,4,128,1,fp8,fp8,0,0.2824160059293111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,8,128,1,float16,fp8,0,0.3083039919535319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,8,128,1,float16,float16,0,0.3085599939028422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,96,8,128,1,fp8,fp8,0,0.29420799016952515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,1,128,1,float16,fp8,0,0.16269866625467935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,1,128,1,float16,float16,0,0.16267200311024985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,96,128,1,float16,float16,0,0.2172213395436605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,1,128,1,fp8,fp8,0,0.1498026649157206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,96,128,1,float16,fp8,0,0.20941867431004843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,96,128,1,fp8,fp8,0,0.18872000773747763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,2,128,1,float16,float16,0,0.1625546713670095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,2,128,1,float16,fp8,0,0.16447466611862183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,2,128,1,fp8,fp8,0,0.15017066399256387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,4,128,1,float16,float16,0,0.1641866664091746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,4,128,1,float16,fp8,0,0.16546666622161865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,4,128,1,fp8,fp8,0,0.1516480048497518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,8,128,1,float16,float16,0,0.16541332999865213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,8,128,1,fp8,fp8,0,0.153957337141037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,96,8,128,1,float16,fp8,0,0.1660266617933909
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,1,128,1,float16,float16,0,0.09161600470542908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,96,128,1,float16,fp8,0,0.13217066725095114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,1,128,1,float16,fp8,0,0.09166399637858073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,96,128,1,fp8,fp8,0,0.10355200370152791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,1,128,1,fp8,fp8,0,0.08286400139331818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,96,128,1,float16,float16,0,0.13595733046531677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,2,128,1,float16,fp8,0,0.0916373332341512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,2,128,1,float16,float16,0,0.09120532870292664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,4,128,1,float16,float16,0,0.09185066819190979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,4,128,1,fp8,fp8,0,0.08403733372688293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,2,128,1,fp8,fp8,0,0.08380800485610962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,4,128,1,float16,fp8,0,0.092357337474823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,8,128,1,float16,fp8,0,0.09297066926956177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,1,128,1,float16,float16,0,0.052442664901415505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,8,128,1,fp8,fp8,0,0.08664533495903015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,96,128,1,fp8,fp8,0,0.06146133442719778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,96,128,1,float16,float16,0,0.06862933437029521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,96,8,128,1,float16,float16,0,0.09258133172988892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,96,128,1,float16,fp8,0,0.0666240006685257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,1,128,1,float16,fp8,0,0.052255998055140175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,1,128,1,fp8,fp8,0,0.04841599861780802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,2,128,1,float16,float16,0,0.05227200190226237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,2,128,1,float16,fp8,0,0.052853330969810486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,2,128,1,fp8,fp8,0,0.04857600231965383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,4,128,1,float16,fp8,0,0.053397332628568016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,4,128,1,fp8,fp8,0,0.0489279975493749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,4,128,1,float16,float16,0,0.0525546669960022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,8,128,1,float16,float16,0,0.05385600030422211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,8,128,1,fp8,fp8,0,0.050213331977526345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,96,8,128,1,float16,fp8,0,0.05442133545875549
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,1,128,1,float16,float16,0,3.956266721089681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,1,128,1,fp8,fp8,0,3.691376050313314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,2,128,1,float16,fp8,0,3.929226557413737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,2,128,1,fp8,fp8,0,3.708160082499186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,1,128,1,float16,fp8,0,3.9292319615681968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,2,128,1,float16,float16,0,3.965973218282064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,4,128,1,float16,float16,0,4.016565322875977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,4,128,1,float16,fp8,0,3.9503841400146484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,4,128,1,fp8,fp8,0,3.7861865361531577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,8,128,1,float16,float16,0,4.124613444010417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,8,128,1,float16,fp8,0,4.193258603413899
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,96,8,128,1,fp8,fp8,0,4.012501398722331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,1,128,1,float16,float16,0,1.9330986340840657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,1,128,1,float16,fp8,0,1.91428804397583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,96,128,1,float16,float16,0,2.884741465250651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,2,128,1,float16,float16,0,1.943413257598877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,1,128,1,fp8,fp8,0,1.8226826985677083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,96,128,1,float16,fp8,0,2.7481867472330728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,2,128,1,float16,fp8,0,1.9446880022684734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,2,128,1,fp8,fp8,0,1.8619680404663086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,4,128,1,float16,float16,0,1.9736693700154622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,4,128,1,float16,fp8,0,1.9633599917093914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,4,128,1,fp8,fp8,0,1.8940320014953613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,96,128,1,fp8,fp8,0,2.4773279825846353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,8,128,1,float16,float16,0,2.0566293398539224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,8,128,1,float16,fp8,0,2.052896022796631
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,1,128,1,float16,fp8,0,0.9676427046457926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,1,128,1,float16,float16,0,0.9681759675343832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,96,8,128,1,fp8,fp8,0,2.007967948913574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,1,128,1,fp8,fp8,0,0.9145119984944662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,96,128,1,float16,float16,0,1.4339733123779297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,96,128,1,float16,fp8,0,1.3755520184834797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,96,128,1,fp8,fp8,0,1.2286240259806316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,2,128,1,float16,float16,0,0.9765386581420898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,2,128,1,fp8,fp8,0,0.9316533406575521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,2,128,1,float16,fp8,0,0.9820533593495687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,4,128,1,float16,float16,0,0.98198930422465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,4,128,1,float16,fp8,0,0.984282652537028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,4,128,1,fp8,fp8,0,0.9508533477783203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,8,128,1,float16,float16,0,1.011562665303548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,8,128,1,float16,fp8,0,0.9931840101877848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,96,8,128,1,fp8,fp8,0,1.0132053693135579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,1,128,1,float16,float16,0,0.49409600098927814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,96,128,1,float16,float16,0,0.7279893557230631
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,1,128,1,float16,fp8,0,0.4958719809850057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,1,128,1,fp8,fp8,0,0.46194132169087726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,2,128,1,float16,float16,0,0.49941333134969074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,2,128,1,float16,fp8,0,0.5011093219121298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,2,128,1,fp8,fp8,0,0.4726826747258504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,96,128,1,float16,fp8,0,0.6927146911621094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,4,128,1,float16,float16,0,0.5007733503977457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,4,128,1,float16,fp8,0,0.5024533271789551
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,96,128,1,fp8,fp8,0,0.6289439996083578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,4,128,1,fp8,fp8,0,0.4750346740086873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,8,128,1,float16,float16,0,0.5087306499481201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,1,128,1,float16,float16,0,0.26065067450205487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,8,128,1,float16,fp8,0,0.5086666742960612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,96,8,128,1,fp8,fp8,0,0.5125226577123007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,96,128,1,float16,float16,0,0.3770453135172526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,96,128,1,float16,fp8,0,0.35947732130686444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,96,128,1,fp8,fp8,0,0.3173440098762512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,1,128,1,float16,fp8,0,0.25918400287628174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,1,128,1,fp8,fp8,0,0.243504007657369
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,2,128,1,float16,float16,0,0.26046933730443317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,2,128,1,float16,fp8,0,0.26184000571568805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,2,128,1,fp8,fp8,0,0.24877333641052246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,4,128,1,float16,fp8,0,0.26372265815734863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,4,128,1,fp8,fp8,0,0.24812267223993936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,4,128,1,float16,float16,0,0.2608853379885356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,8,128,1,float16,fp8,0,0.26497600475947064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,8,128,1,float16,float16,0,0.2661386728286743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,96,8,128,1,fp8,fp8,0,0.2617493271827698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,96,128,1,float16,float16,0,0.20057066281636557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,96,128,1,float16,fp8,0,0.1925119956334432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,96,128,1,fp8,fp8,0,0.17125332355499268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,1,128,1,float16,fp8,0,0.14011733730634054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,1,128,1,fp8,fp8,0,0.13512000441551208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,1,128,1,float16,float16,0,0.14060800274213156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,2,128,1,float16,float16,0,0.1411626636981964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,2,128,1,float16,fp8,0,0.14121066530545553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,2,128,1,fp8,fp8,0,0.13458133737246195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,4,128,1,float16,fp8,0,0.14245333274205527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,4,128,1,fp8,fp8,0,0.13707199692726135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,4,128,1,float16,float16,0,0.14166399836540222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,8,128,1,float16,fp8,0,0.1439253290494283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,8,128,1,float16,float16,0,0.14296533664067587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,96,8,128,1,fp8,fp8,0,0.1402773360411326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,1,128,1,float16,fp8,0,0.08224533498287201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,1,128,1,float16,float16,0,0.08161066472530365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,96,128,1,float16,fp8,0,0.12056533495585124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,96,128,1,fp8,fp8,0,0.09896533687909444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,1,128,1,fp8,fp8,0,0.07484266658624013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,96,128,1,float16,float16,0,0.12467733025550842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,2,128,1,float16,float16,0,0.08167999982833862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,2,128,1,float16,fp8,0,0.08236800134181976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,4,128,1,float16,float16,0,0.081386665503184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,4,128,1,float16,fp8,0,0.08205333352088928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,2,128,1,fp8,fp8,0,0.0753119985262553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,4,128,1,fp8,fp8,0,0.07548266649246216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,8,128,1,float16,float16,0,0.08244266609350841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,8,128,1,float16,fp8,0,0.08317866424719493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,96,8,128,1,fp8,fp8,0,0.07761066655317943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,1,128,1,float16,float16,0,0.0473280002673467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,96,128,1,float16,fp8,0,0.0621973325808843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,96,128,1,float16,float16,0,0.0656160016854604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,96,128,1,fp8,fp8,0,0.057215998570124306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,2,128,1,float16,fp8,0,0.047295997540156044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,1,128,1,fp8,fp8,0,0.04353066782156626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,2,128,1,float16,float16,0,0.04770133395989736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,1,128,1,float16,fp8,0,0.04806933303674062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,2,128,1,fp8,fp8,0,0.042837331692377724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,4,128,1,float16,float16,0,0.04785599807898203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,4,128,1,fp8,fp8,0,0.04268800218900045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,4,128,1,float16,fp8,0,0.04818133513132731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,8,128,1,float16,float16,0,0.04840533435344696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,8,128,1,float16,fp8,0,0.04860266546408335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,96,8,128,1,fp8,fp8,0,0.04384533564249674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,96,128,1,float16,float16,0,0.038389332592487335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,96,128,1,float16,fp8,0,0.0390079990029335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,96,128,1,fp8,fp8,0,0.037647999823093414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,1,128,1,float16,fp8,0,0.03457599878311157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,1,128,1,float16,float16,0,0.03433600068092346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,1,128,1,fp8,fp8,0,0.03145066648721695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,2,128,1,float16,fp8,0,0.03495466709136963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,2,128,1,float16,float16,0,0.03416533271471659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,2,128,1,fp8,fp8,0,0.03161599983771642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,4,128,1,float16,float16,0,0.034501334031422935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,4,128,1,float16,fp8,0,0.03417599946260452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,4,128,1,fp8,fp8,0,0.031712000568707786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,8,128,1,float16,float16,0,0.03453333427508672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,8,128,1,float16,fp8,0,0.03503466645876566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,96,8,128,1,fp8,fp8,0,0.0323840007185936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,1,128,1,float16,fp8,0,1.7571412722269695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,1,128,1,fp8,fp8,0,1.8404000600179036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,2,128,1,float16,fp8,0,1.7775360743204753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,2,128,1,fp8,fp8,0,1.8529653549194336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,1,128,1,float16,float16,0,1.7726186116536458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,4,128,1,float16,float16,0,1.8387519518534343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,2,128,1,float16,float16,0,1.7872427304585774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,4,128,1,fp8,fp8,0,1.8804480234781902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,8,128,1,float16,float16,0,1.8828320503234863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,8,128,1,float16,fp8,0,1.895616054534912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,8,128,1,fp8,fp8,0,2.00273068745931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,96,4,128,1,float16,fp8,0,1.8060266176859539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,96,128,1,float16,fp8,0,1.441754659016927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,1,128,1,float16,float16,0,0.8885440031687418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,96,128,1,float16,float16,0,1.485914707183838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,1,128,1,float16,fp8,0,0.8865013122558594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,2,128,1,float16,float16,0,0.8996000289916992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,1,128,1,fp8,fp8,0,0.9169119993845621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,2,128,1,float16,fp8,0,0.9007840156555176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,2,128,1,fp8,fp8,0,0.9187626838684082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,4,128,1,float16,float16,0,0.9027573267618815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,4,128,1,float16,fp8,0,0.9033333460489908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,4,128,1,fp8,fp8,0,0.9422346750895182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,8,128,1,float16,float16,0,0.9376426537831625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,96,128,1,fp8,fp8,0,1.232208013534546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,1,128,1,float16,float16,0,0.4561013380686442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,1,128,1,float16,fp8,0,0.45670398076375324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,8,128,1,float16,fp8,0,0.9201866785685221
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,96,8,128,1,fp8,fp8,0,1.0011626879374187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,96,128,1,float16,float16,0,0.7481919924418131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,1,128,1,fp8,fp8,0,0.4581120014190674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,96,128,1,float16,fp8,0,0.7152853012084961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,96,128,1,fp8,fp8,0,0.6173813343048096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,2,128,1,float16,float16,0,0.4615519841512044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,2,128,1,float16,fp8,0,0.4617653290430705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,2,128,1,fp8,fp8,0,0.46485865116119385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,4,128,1,float16,fp8,0,0.46354134877522785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,4,128,1,fp8,fp8,0,0.4710880120595296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,8,128,1,float16,float16,0,0.4698186715443929
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,8,128,1,float16,fp8,0,0.4687306483586629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,8,128,1,fp8,fp8,0,0.5104639927546183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,96,4,128,1,float16,float16,0,0.4617653290430705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,96,128,1,float16,float16,0,0.38847466309865314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,1,128,1,float16,float16,0,0.24117867151896158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,1,128,1,float16,fp8,0,0.2412853240966797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,1,128,1,fp8,fp8,0,0.24034667015075684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,2,128,1,float16,float16,0,0.24181866645812988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,2,128,1,float16,fp8,0,0.24331732590993246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,96,128,1,float16,fp8,0,0.37141335010528564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,2,128,1,fp8,fp8,0,0.24487467606862387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,4,128,1,float16,float16,0,0.24415467182795206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,4,128,1,float16,fp8,0,0.24529600143432617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,96,128,1,fp8,fp8,0,0.3189919988314311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,4,128,1,fp8,fp8,0,0.24605333805084229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,8,128,1,float16,float16,0,0.24863467613855997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,8,128,1,float16,fp8,0,0.24743467569351196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,96,8,128,1,fp8,fp8,0,0.26076799631118774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,96,128,1,float16,float16,0,0.2083573341369629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,1,128,1,float16,float16,0,0.1320693294207255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,1,128,1,float16,fp8,0,0.13277332981427512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,96,128,1,float16,fp8,0,0.1993066668510437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,96,128,1,fp8,fp8,0,0.17128000656763712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,1,128,1,fp8,fp8,0,0.13329600294431052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,2,128,1,float16,float16,0,0.13329066832860312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,2,128,1,float16,fp8,0,0.1338933308919271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,2,128,1,fp8,fp8,0,0.13402133186658224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,4,128,1,float16,float16,0,0.13437333703041077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,4,128,1,float16,fp8,0,0.13569600383440653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,4,128,1,fp8,fp8,0,0.13666133085886636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,8,128,1,float16,fp8,0,0.13678933183352152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,8,128,1,float16,float16,0,0.13590932885805765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,96,8,128,1,fp8,fp8,0,0.13831466436386108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,96,128,1,float16,float16,0,0.11682666341463725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,1,128,1,float16,fp8,0,0.07708799839019775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,1,128,1,float16,float16,0,0.07723199824492137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,1,128,1,fp8,fp8,0,0.07517333328723907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,96,128,1,float16,fp8,0,0.11380267143249512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,96,128,1,fp8,fp8,0,0.0953653355439504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,2,128,1,float16,float16,0,0.07729066908359528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,2,128,1,float16,fp8,0,0.0773173322280248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,2,128,1,fp8,fp8,0,0.07437866429487865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,4,128,1,float16,fp8,0,0.07758933305740356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,4,128,1,float16,float16,0,0.07790400087833405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,4,128,1,fp8,fp8,0,0.07538666824499766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,8,128,1,float16,float16,0,0.07840533554553986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,8,128,1,float16,fp8,0,0.07829866806666057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,1,128,1,float16,float16,0,0.04532266656557719
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,1,128,1,float16,fp8,0,0.04553066690762838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,96,8,128,1,fp8,fp8,0,0.07724800209204356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,96,128,1,float16,float16,0,0.06474133332570393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,96,128,1,float16,fp8,0,0.061664000153541565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,96,128,1,fp8,fp8,0,0.05719999969005585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,1,128,1,fp8,fp8,0,0.042549331982930504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,2,128,1,float16,float16,0,0.04502399762471517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,2,128,1,float16,fp8,0,0.0458186666170756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,4,128,1,float16,float16,0,0.045509333411852516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,2,128,1,fp8,fp8,0,0.042410666743914284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,4,128,1,float16,fp8,0,0.04530133306980133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,4,128,1,fp8,fp8,0,0.04251733422279358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,8,128,1,float16,float16,0,0.04602666695912679
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,8,128,1,float16,fp8,0,0.04587733248869578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,96,8,128,1,fp8,fp8,0,0.044346665342648826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,96,128,1,float16,float16,0,0.036917333801587425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,96,128,1,float16,fp8,0,0.03687999894221624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,1,128,1,float16,float16,0,0.03402666747570038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,96,128,1,fp8,fp8,0,0.036874666810035706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,1,128,1,float16,fp8,0,0.03429333368937174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,1,128,1,fp8,fp8,0,0.03127466638882955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,2,128,1,float16,fp8,0,0.03398933261632919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,2,128,1,fp8,fp8,0,0.03160000095764796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,2,128,1,float16,float16,0,0.03358400116364161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,4,128,1,float16,float16,0,0.03392533212900162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,4,128,1,float16,fp8,0,0.03420799970626831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,4,128,1,fp8,fp8,0,0.03149333347876867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,8,128,1,float16,fp8,0,0.03432533393303553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,8,128,1,float16,float16,0,0.03410666684309641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,96,8,128,1,fp8,fp8,0,0.031632001201311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,96,128,1,float16,float16,0,0.025114665428797405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,1,128,1,float16,float16,0,0.023200000325838726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,96,128,1,fp8,fp8,0,0.02438933402299881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,96,128,1,float16,fp8,0,0.025957333544890087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,1,128,1,float16,fp8,0,0.024138666689395905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,1,128,1,fp8,fp8,0,0.022842665513356526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,2,128,1,float16,float16,0,0.023584000766277313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,2,128,1,float16,fp8,0,0.023898666103680927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,2,128,1,fp8,fp8,0,0.0230880007147789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,4,128,1,float16,float16,0,0.023520000278949738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,4,128,1,float16,fp8,0,0.024117333193620045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,4,128,1,fp8,fp8,0,0.02327466756105423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,8,128,1,float16,float16,0,0.023973333338896435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,8,128,1,float16,fp8,0,0.024143998821576435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,96,8,128,1,fp8,fp8,0,0.023285334308942158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,1,128,1,float16,fp8,0,1.1201600233713787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,1,128,1,float16,float16,0,1.1217599709828694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,2,128,1,float16,float16,0,1.1366559664408367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,1,128,1,fp8,fp8,0,1.2194453080495198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,2,128,1,float16,fp8,0,1.1381386915842693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,2,128,1,fp8,fp8,0,1.2277493476867676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,4,128,1,float16,float16,0,1.1380799611409504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,4,128,1,float16,fp8,0,1.140064001083374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,4,128,1,fp8,fp8,0,1.248469352722168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,8,128,1,fp8,fp8,0,1.3257760206858318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,8,128,1,float16,float16,0,1.1814506848653157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,96,8,128,1,float16,fp8,0,1.194111982981364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,96,128,1,float16,fp8,0,0.7173173427581787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,96,128,1,float16,float16,0,0.7569546699523926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,1,128,1,float16,float16,0,0.571999986966451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,1,128,1,float16,fp8,0,0.5708693265914917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,1,128,1,fp8,fp8,0,0.6090720097223917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,2,128,1,float16,float16,0,0.5767946640650431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,2,128,1,float16,fp8,0,0.5785173177719116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,4,128,1,float16,float16,0,0.5773493448893229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,4,128,1,fp8,fp8,0,0.6295200188954672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,4,128,1,float16,fp8,0,0.5788106520970663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,2,128,1,fp8,fp8,0,0.6208213170369467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,96,128,1,fp8,fp8,0,0.773850679397583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,8,128,1,float16,float16,0,0.5870240132013956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,8,128,1,float16,fp8,0,0.5842719872792562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,1,128,1,float16,float16,0,0.2958986759185791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,1,128,1,float16,fp8,0,0.2956533432006836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,96,8,128,1,fp8,fp8,0,0.6727360089619955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,1,128,1,fp8,fp8,0,0.3160746693611145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,96,128,1,float16,float16,0,0.3895039955774943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,96,128,1,float16,fp8,0,0.3703840176264445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,2,128,1,float16,float16,0,0.2972000042597453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,2,128,1,float16,fp8,0,0.2976800004641215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,2,128,1,fp8,fp8,0,0.32467732826868695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,96,128,1,fp8,fp8,0,0.3906773328781128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,4,128,1,float16,float16,0,0.29783467451731366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,4,128,1,float16,fp8,0,0.2988479932149251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,4,128,1,fp8,fp8,0,0.3243786692619324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,8,128,1,float16,float16,0,0.3035306731859843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,8,128,1,float16,fp8,0,0.30208534002304077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,96,8,128,1,fp8,fp8,0,0.3404159943262736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,1,128,1,float16,float16,0,0.15738133589426676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,96,128,1,float16,float16,0,0.2081600030263265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,1,128,1,float16,fp8,0,0.15704533457756042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,1,128,1,fp8,fp8,0,0.17196800311406454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,2,128,1,float16,float16,0,0.1577600042025248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,96,128,1,fp8,fp8,0,0.2067413330078125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,2,128,1,float16,fp8,0,0.1583146651585897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,96,128,1,float16,fp8,0,0.19830399751663208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,2,128,1,fp8,fp8,0,0.17252800861994425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,4,128,1,float16,float16,0,0.15919466813405356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,4,128,1,float16,fp8,0,0.15934399763743082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,4,128,1,fp8,fp8,0,0.1739413340886434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,8,128,1,float16,float16,0,0.15990400314331055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,8,128,1,float16,fp8,0,0.16105600198109946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,1,128,1,float16,float16,0,0.0862559974193573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,1,128,1,float16,fp8,0,0.08664533495903015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,96,128,1,float16,float16,0,0.1128000020980835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,96,128,1,float16,fp8,0,0.10904533664385478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,96,128,1,fp8,fp8,0,0.11516799529393514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,96,8,128,1,fp8,fp8,0,0.1767359972000122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,1,128,1,fp8,fp8,0,0.09515733520189922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,2,128,1,float16,float16,0,0.08672533432642619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,2,128,1,float16,fp8,0,0.08694932858149211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,2,128,1,fp8,fp8,0,0.09457600116729736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,4,128,1,float16,float16,0,0.08684800068537395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,4,128,1,float16,fp8,0,0.08844266335169475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,4,128,1,fp8,fp8,0,0.09593066573143005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,8,128,1,float16,float16,0,0.08819199601809184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,8,128,1,float16,fp8,0,0.08826667070388794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,96,8,128,1,fp8,fp8,0,0.09749866525332133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,96,128,1,float16,float16,0,0.06513066589832306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,96,128,1,float16,fp8,0,0.06211199859778086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,96,128,1,fp8,fp8,0,0.06684799989064534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,1,128,1,float16,fp8,0,0.05009600023428599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,1,128,1,float16,float16,0,0.05087999999523163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,1,128,1,fp8,fp8,0,0.05323733389377594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,2,128,1,float16,float16,0,0.04966400067011515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,2,128,1,float16,fp8,0,0.050426666935284935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,2,128,1,fp8,fp8,0,0.05312533179918925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,4,128,1,float16,float16,0,0.050581331054369606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,4,128,1,float16,fp8,0,0.050810664892196655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,4,128,1,fp8,fp8,0,0.05364799996217092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,8,128,1,float16,float16,0,0.050944000482559204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,8,128,1,float16,fp8,0,0.050848002235094704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,96,128,1,float16,float16,0,0.03622400015592575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,96,8,128,1,fp8,fp8,0,0.05442133545875549
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,96,128,1,fp8,fp8,0,0.040591999888420105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,96,128,1,float16,fp8,0,0.035189333061377205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,1,128,1,float16,float16,0,0.03370666752258936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,1,128,1,float16,fp8,0,0.03401600072781245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,2,128,1,float16,float16,0,0.033701332906881966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,1,128,1,fp8,fp8,0,0.03474666674931844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,2,128,1,float16,fp8,0,0.034143999218940735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,2,128,1,fp8,fp8,0,0.034858666360378265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,4,128,1,float16,fp8,0,0.03421333432197571
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,4,128,1,float16,float16,0,0.03396799912055334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,4,128,1,fp8,fp8,0,0.03510933369398117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,8,128,1,float16,float16,0,0.034143999218940735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,8,128,1,fp8,fp8,0,0.03544000039498011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,96,8,128,1,float16,fp8,0,0.034416000048319496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,96,128,1,float16,fp8,0,0.027493332823117573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,96,128,1,float16,float16,0,0.027237333357334137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,96,128,1,fp8,fp8,0,0.0284853329261144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,1,128,1,float16,float16,0,0.025216000775496166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,1,128,1,float16,fp8,0,0.02610666553179423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,1,128,1,fp8,fp8,0,0.026911998788515728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,2,128,1,float16,fp8,0,0.026496000587940216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,2,128,1,fp8,fp8,0,0.027056001126766205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,2,128,1,float16,float16,0,0.026330667237440746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,4,128,1,float16,float16,0,0.026367999613285065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,4,128,1,float16,fp8,0,0.02622399975856145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,8,128,1,float16,float16,0,0.02606400102376938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,4,128,1,fp8,fp8,0,0.02703999976317088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,8,128,1,float16,fp8,0,0.02604266752799352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,96,8,128,1,fp8,fp8,0,0.027482666075229645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,96,128,1,float16,fp8,0,0.020197333147128422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,96,128,1,fp8,fp8,0,0.020661332954963047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,96,128,1,float16,float16,0,0.019808000574509304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,1,128,1,float16,float16,0,0.018853332847356796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,1,128,1,float16,fp8,0,0.018613333503405254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,1,128,1,fp8,fp8,0,0.019530666371186573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,2,128,1,float16,fp8,0,0.01961600035429001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,2,128,1,float16,float16,0,0.01953599974513054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,2,128,1,fp8,fp8,0,0.019973333925008774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,4,128,1,float16,fp8,0,0.019039999693632126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,4,128,1,float16,float16,0,0.019333332777023315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,8,128,1,float16,float16,0,0.018976000448067982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,4,128,1,fp8,fp8,0,0.019600000232458115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,8,128,1,fp8,fp8,0,0.020362666497627895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,96,8,128,1,float16,fp8,0,0.019754666835069656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,1,128,1,float16,float16,0,0.7595360279083252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,1,128,1,float16,fp8,0,0.7606773376464844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,2,128,1,float16,float16,0,0.7731680075327555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,2,128,1,float16,fp8,0,0.7745013236999512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,2,128,1,fp8,fp8,0,0.9715840021769205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,4,128,1,float16,float16,0,0.7865119775136312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,1,128,1,fp8,fp8,0,0.9599253336588541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,4,128,1,float16,fp8,0,0.7880266507466634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,4,128,1,fp8,fp8,0,0.9739786783854166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,8,128,1,float16,float16,0,0.7992106278737386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,8,128,1,float16,fp8,0,0.7992640336354574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,96,8,128,1,fp8,fp8,0,1.0232053597768147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,96,128,1,float16,float16,0,0.48877867062886554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,1,128,1,float16,float16,0,0.39141865571339923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,1,128,1,float16,fp8,0,0.3897706667582194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,1,128,1,fp8,fp8,0,0.49322664737701416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,2,128,1,float16,float16,0,0.39538665612538654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,2,128,1,float16,fp8,0,0.3970613479614258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,96,128,1,fp8,fp8,0,0.5671360095342001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,2,128,1,fp8,fp8,0,0.498639980951945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,4,128,1,float16,float16,0,0.4009759823481242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,96,128,1,float16,fp8,0,0.4703520139058431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,4,128,1,float16,fp8,0,0.40378665924072266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,4,128,1,fp8,fp8,0,0.5039039850234985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,8,128,1,float16,float16,0,0.4099786678949992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,1,128,1,float16,float16,0,0.2037013371785482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,8,128,1,fp8,fp8,0,0.5173173348108927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,96,128,1,float16,float16,0,0.2569920023282369
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,96,128,1,float16,fp8,0,0.24523733059565225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,96,8,128,1,float16,fp8,0,0.41039466857910156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,1,128,1,float16,fp8,0,0.20323199033737183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,96,128,1,fp8,fp8,0,0.2955626646677653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,1,128,1,fp8,fp8,0,0.25939732789993286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,2,128,1,float16,float16,0,0.2076693375905355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,2,128,1,float16,fp8,0,0.20705600579579672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,2,128,1,fp8,fp8,0,0.2623680035273234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,4,128,1,float16,float16,0,0.20961066087086996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,4,128,1,float16,fp8,0,0.20992533365885416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,4,128,1,fp8,fp8,0,0.26173333326975506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,8,128,1,float16,float16,0,0.2143519918123881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,8,128,1,float16,fp8,0,0.2139893372853597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,96,8,128,1,fp8,fp8,0,0.2645226716995239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,1,128,1,float16,float16,0,0.10839466253916423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,96,128,1,float16,float16,0,0.1360373298327128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,1,128,1,float16,fp8,0,0.10949333508809407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,96,128,1,fp8,fp8,0,0.15930666526158652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,96,128,1,float16,fp8,0,0.1316426694393158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,1,128,1,fp8,fp8,0,0.14046933253606161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,2,128,1,float16,float16,0,0.10872000455856323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,2,128,1,float16,fp8,0,0.10869333148002625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,2,128,1,fp8,fp8,0,0.1418560047944387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,4,128,1,float16,float16,0,0.10954667131106059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,4,128,1,float16,fp8,0,0.11028266946474712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,4,128,1,fp8,fp8,0,0.14257599910100302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,8,128,1,float16,float16,0,0.1095360020796458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,8,128,1,float16,fp8,0,0.11010666688283284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,96,8,128,1,fp8,fp8,0,0.14472533265749613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,96,128,1,float16,float16,0,0.07634666562080383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,1,128,1,float16,float16,0,0.060965334375699363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,1,128,1,float16,fp8,0,0.060991997520128884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,96,128,1,fp8,fp8,0,0.08980799714724223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,96,128,1,float16,fp8,0,0.07363733152548473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,1,128,1,fp8,fp8,0,0.07666666805744171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,2,128,1,float16,float16,0,0.06121066709359487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,2,128,1,float16,fp8,0,0.06090133388837179
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,2,128,1,fp8,fp8,0,0.07684800028800964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,4,128,1,float16,float16,0,0.06101333101590475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,4,128,1,float16,fp8,0,0.06181333462397257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,4,128,1,fp8,fp8,0,0.07693866888682048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,8,128,1,float16,float16,0,0.06226666768391927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,8,128,1,float16,fp8,0,0.062080000837643944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,96,8,128,1,fp8,fp8,0,0.07915199796358745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,96,128,1,float16,float16,0,0.04215999941031138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,96,128,1,float16,fp8,0,0.0415786678592364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,1,128,1,float16,float16,0,0.039408000806967415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,1,128,1,float16,fp8,0,0.039664000272750854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,96,128,1,fp8,fp8,0,0.052895997961362205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,1,128,1,fp8,fp8,0,0.0468746672074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,2,128,1,float16,float16,0,0.03931200007597605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,2,128,1,fp8,fp8,0,0.04698666433493296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,2,128,1,float16,fp8,0,0.03972800076007843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,4,128,1,float16,float16,0,0.039349332451820374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,4,128,1,fp8,fp8,0,0.04728533327579498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,4,128,1,float16,fp8,0,0.03990933299064636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,8,128,1,float16,fp8,0,0.039066667358080544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,8,128,1,float16,float16,0,0.0391839991013209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,96,128,1,float16,float16,0,0.028207999964555103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,96,128,1,fp8,fp8,0,0.03305066625277201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,96,8,128,1,fp8,fp8,0,0.047210668524106346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,1,128,1,float16,float16,0,0.027189334233601887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,1,128,1,float16,fp8,0,0.026762666801611584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,96,128,1,float16,fp8,0,0.028351999819278717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,1,128,1,fp8,fp8,0,0.03051200012365977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,2,128,1,float16,fp8,0,0.02773866554101308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,2,128,1,float16,float16,0,0.027248000105222065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,4,128,1,float16,fp8,0,0.026874666412671406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,2,128,1,fp8,fp8,0,0.03133866687615713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,4,128,1,float16,float16,0,0.027189334233601887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,4,128,1,fp8,fp8,0,0.031184000273545582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,8,128,1,float16,float16,0,0.026895999908447266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,8,128,1,float16,fp8,0,0.02737066646416982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,96,128,1,float16,float16,0,0.021509334444999695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,96,8,128,1,fp8,fp8,0,0.031066666046778362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,96,128,1,float16,fp8,0,0.021914665897687275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,1,128,1,float16,float16,0,0.02130666623512904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,96,128,1,fp8,fp8,0,0.024634666740894318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,1,128,1,fp8,fp8,0,0.023941333095232647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,1,128,1,float16,fp8,0,0.021344001094500225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,2,128,1,float16,float16,0,0.021327999730904896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,2,128,1,fp8,fp8,0,0.02385599911212921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,2,128,1,float16,fp8,0,0.02146666745344798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,4,128,1,float16,float16,0,0.021664001047611237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,4,128,1,fp8,fp8,0,0.024442667762438457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,4,128,1,float16,fp8,0,0.022287999590237934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,8,128,1,float16,float16,0,0.02162133405605952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,8,128,1,float16,fp8,0,0.022085333863894146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,96,8,128,1,fp8,fp8,0,0.023610666394233704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,96,128,1,float16,float16,0,0.017850667238235474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,96,128,1,float16,fp8,0,0.01860800012946129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,96,128,1,fp8,fp8,0,0.020117333779732387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,1,128,1,fp8,fp8,0,0.019082666685183842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,1,128,1,float16,fp8,0,0.017573333034912746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,1,128,1,float16,float16,0,0.01743999992807706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,2,128,1,float16,float16,0,0.017594666530688603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,2,128,1,float16,fp8,0,0.017845333864291508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,2,128,1,fp8,fp8,0,0.019343999524911244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,4,128,1,float16,fp8,0,0.018298666924238205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,4,128,1,float16,float16,0,0.017909333109855652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,4,128,1,fp8,fp8,0,0.019152000546455383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,8,128,1,float16,fp8,0,0.017925333231687546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,8,128,1,float16,float16,0,0.017717332889636356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,96,8,128,1,fp8,fp8,0,0.01953599974513054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,1,128,1,float16,float16,0,0.6459093491236368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,1,128,1,fp8,fp8,0,0.847653309504191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,1,128,1,float16,fp8,0,0.645029346148173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,2,128,1,float16,float16,0,0.6516373157501221
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,2,128,1,float16,fp8,0,0.6508320172627767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,4,128,1,float16,float16,0,0.6555893421173096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,2,128,1,fp8,fp8,0,0.8517599900563558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,8,128,1,float16,fp8,0,0.6633119980494181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,4,128,1,fp8,fp8,0,0.8610080083211263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,8,128,1,float16,float16,0,0.6626826524734497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,4,128,1,float16,fp8,0,0.6567200024922689
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,96,8,128,1,fp8,fp8,0,0.8773386478424072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,96,128,1,float16,float16,0,0.394869327545166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,1,128,1,float16,float16,0,0.3304906686147054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,1,128,1,float16,fp8,0,0.3309066692988078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,2,128,1,float16,float16,0,0.33541866143544513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,1,128,1,fp8,fp8,0,0.4367893139521281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,2,128,1,float16,fp8,0,0.3352106809616089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,96,128,1,float16,fp8,0,0.39695998032887775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,96,128,1,fp8,fp8,0,0.46961065133412677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,2,128,1,fp8,fp8,0,0.4389386574427287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,4,128,1,float16,float16,0,0.33579198519388836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,4,128,1,float16,fp8,0,0.33632532755533856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,4,128,1,fp8,fp8,0,0.4381279945373535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,8,128,1,float16,float16,0,0.3406613270441691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,8,128,1,fp8,fp8,0,0.4434613386789958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,96,8,128,1,float16,fp8,0,0.34141333897908527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,1,128,1,float16,float16,0,0.17177599668502808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,96,128,1,float16,float16,0,0.18988800048828125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,1,128,1,float16,fp8,0,0.17246399323145548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,96,128,1,fp8,fp8,0,0.24653865893681845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,1,128,1,fp8,fp8,0,0.23203200101852417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,96,128,1,float16,fp8,0,0.1851466695467631
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,2,128,1,float16,float16,0,0.172106663386027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,2,128,1,float16,fp8,0,0.17278399070103964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,2,128,1,fp8,fp8,0,0.23348265886306763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,4,128,1,float16,float16,0,0.17261866728464761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,4,128,1,float16,fp8,0,0.17283199230829874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,4,128,1,fp8,fp8,0,0.23258666197458902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,8,128,1,float16,float16,0,0.17380799849828085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,8,128,1,fp8,fp8,0,0.23410133520762125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,96,128,1,float16,float16,0,0.1037546694278717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,1,128,1,float16,float16,0,0.09269866347312927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,1,128,1,float16,fp8,0,0.09310932954152425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,96,128,1,float16,fp8,0,0.09964266419410706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,96,8,128,1,float16,fp8,0,0.17404266198476157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,96,128,1,fp8,fp8,0,0.13573867082595825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,1,128,1,fp8,fp8,0,0.12314132849375407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,2,128,1,fp8,fp8,0,0.12334932883580525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,2,128,1,float16,float16,0,0.09377599755922954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,2,128,1,float16,fp8,0,0.09302399555842082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,4,128,1,float16,float16,0,0.09294399619102478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,4,128,1,fp8,fp8,0,0.12381333112716675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,4,128,1,float16,fp8,0,0.09418132901191711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,8,128,1,float16,float16,0,0.09408533573150635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,8,128,1,float16,fp8,0,0.09444266557693481
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,96,8,128,1,fp8,fp8,0,0.1260586678981781
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,96,128,1,float16,float16,0,0.05578133463859558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,96,128,1,float16,fp8,0,0.05505066613356272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,96,128,1,fp8,fp8,0,0.0754559983809789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,1,128,1,float16,float16,0,0.05561600128809611
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,1,128,1,float16,fp8,0,0.056277334690093994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,1,128,1,fp8,fp8,0,0.07066133121649425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,2,128,1,float16,float16,0,0.05601066847642263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,2,128,1,float16,fp8,0,0.055919999877611794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,2,128,1,fp8,fp8,0,0.0708426684141159
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,4,128,1,float16,float16,0,0.05580799778302511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,4,128,1,float16,fp8,0,0.055770665407180786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,4,128,1,fp8,fp8,0,0.07121600210666656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,8,128,1,float16,float16,0,0.05551999807357788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,8,128,1,float16,fp8,0,0.05554133156935374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,96,8,128,1,fp8,fp8,0,0.07137066622575124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,96,128,1,float16,fp8,0,0.03606933355331421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,96,128,1,float16,float16,0,0.03570133447647095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,96,128,1,fp8,fp8,0,0.04424533247947693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,1,128,1,float16,float16,0,0.0353973334034284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,1,128,1,float16,fp8,0,0.035391998787721
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,2,128,1,float16,fp8,0,0.03589866558710734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,1,128,1,fp8,fp8,0,0.04292800029118856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,2,128,1,float16,float16,0,0.03570666660865148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,2,128,1,fp8,fp8,0,0.043653334180514015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,4,128,1,float16,fp8,0,0.035973332822322845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,4,128,1,fp8,fp8,0,0.04308266441027323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,8,128,1,float16,float16,0,0.03581333408753077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,4,128,1,float16,float16,0,0.03587199995915095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,8,128,1,float16,fp8,0,0.03600533306598663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,96,8,128,1,fp8,fp8,0,0.04331199824810028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,96,128,1,float16,float16,0,0.024826665719350178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,96,128,1,float16,fp8,0,0.025616000096003216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,96,128,1,fp8,fp8,0,0.028927999238173168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,1,128,1,float16,float16,0,0.02385599911212921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,1,128,1,fp8,fp8,0,0.028368001182874043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,2,128,1,float16,float16,0,0.024906667570273083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,1,128,1,float16,fp8,0,0.0242399995525678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,2,128,1,float16,fp8,0,0.024885334074497223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,4,128,1,float16,float16,0,0.024634666740894318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,2,128,1,fp8,fp8,0,0.02845333268245061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,4,128,1,float16,fp8,0,0.024357333779335022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,4,128,1,fp8,fp8,0,0.029029332101345062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,8,128,1,float16,float16,0,0.02421333392461141
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,8,128,1,float16,fp8,0,0.02510400116443634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,96,128,1,float16,float16,0,0.020736000190178554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,96,8,128,1,fp8,fp8,0,0.027930667002995808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,96,128,1,fp8,fp8,0,0.023562667270501454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,96,128,1,float16,fp8,0,0.021477334201335907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,1,128,1,float16,float16,0,0.021087999145189922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,1,128,1,float16,fp8,0,0.02086399992307027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,1,128,1,fp8,fp8,0,0.02330133318901062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,2,128,1,float16,float16,0,0.020442667106787365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,2,128,1,fp8,fp8,0,0.02311466634273529
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,2,128,1,float16,fp8,0,0.02091199904680252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,4,128,1,float16,float16,0,0.02091199904680252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,4,128,1,float16,fp8,0,0.02144533395767212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,4,128,1,fp8,fp8,0,0.023386667172114056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,8,128,1,float16,float16,0,0.021087999145189922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,8,128,1,float16,fp8,0,0.021136000752449036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,96,8,128,1,fp8,fp8,0,0.023418667415777843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,96,128,1,float16,float16,0,0.017279999951521557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,96,128,1,float16,fp8,0,0.017674667139848072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,96,128,1,fp8,fp8,0,0.01933866615096728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,1,128,1,float16,fp8,0,0.017488000293572743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,1,128,1,float16,float16,0,0.01703466723362605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,1,128,1,fp8,fp8,0,0.01882133384545644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,2,128,1,float16,float16,0,0.017018667111794155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,2,128,1,fp8,fp8,0,0.018917333334684372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,4,128,1,float16,fp8,0,0.017477333545684814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,4,128,1,fp8,fp8,0,0.018805333723624546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,2,128,1,float16,fp8,0,0.01764800027012825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,4,128,1,float16,float16,0,0.017360000560681026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,8,128,1,float16,float16,0,0.01687466725707054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,8,128,1,float16,fp8,0,0.017717332889636356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,96,8,128,1,fp8,fp8,0,0.019050666441520054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,1,128,1,float16,float16,0,75.00437418619792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,2,128,1,fp8,fp8,0,45.067779541015625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,1,128,1,float16,fp8,0,74.07696533203125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,1,128,1,fp8,fp8,0,44.131978352864586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,2,128,1,float16,fp8,0,76.67442830403645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,2,128,1,float16,float16,0,76.04613240559895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,4,128,1,float16,float16,0,75.79200236002605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,4,128,1,fp8,fp8,0,44.93530782063802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,4,128,1,float16,fp8,0,76.36996968587239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,8,128,1,fp8,fp8,0,44.3578135172526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,8,128,1,float16,float16,0,76.806396484375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,64,8,128,1,float16,fp8,0,77.82593790690105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,64,128,1,float16,float16,0,33.66948699951172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,1,128,1,float16,float16,0,33.73303985595703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,64,128,1,float16,fp8,0,33.09210713704427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,1,128,1,float16,fp8,0,33.61234792073568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,1,128,1,fp8,fp8,0,22.099652608235676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,2,128,1,float16,float16,0,33.853424072265625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,2,128,1,float16,fp8,0,33.394816080729164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,2,128,1,fp8,fp8,0,22.21332295735677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,4,128,1,float16,float16,0,33.35809071858724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,64,128,1,fp8,fp8,0,23.0798823038737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,4,128,1,float16,fp8,0,33.76995086669922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,4,128,1,fp8,fp8,0,21.894139607747395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,8,128,1,float16,float16,0,33.99609629313151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,8,128,1,float16,fp8,0,33.700948079427086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,64,8,128,1,fp8,fp8,0,22.171348571777344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,1,128,1,float16,fp8,0,16.962298075358074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,1,128,1,float16,float16,0,16.864852905273438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,64,128,1,float16,float16,0,16.81605275472005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,1,128,1,fp8,fp8,0,11.220021565755209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,2,128,1,float16,float16,0,16.887301127115887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,64,128,1,float16,fp8,0,17.30728022257487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,64,128,1,fp8,fp8,0,11.5623410542806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,2,128,1,float16,fp8,0,16.960987091064453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,2,128,1,fp8,fp8,0,11.103338877360025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,4,128,1,fp8,fp8,0,11.251232147216797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,4,128,1,float16,float16,0,16.78999964396159
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,4,128,1,float16,fp8,0,16.899439493815105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,8,128,1,float16,float16,0,17.497552235921223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,8,128,1,float16,fp8,0,17.075946807861328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,64,8,128,1,fp8,fp8,0,11.289738972981771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,1,128,1,float16,float16,0,8.676634470621744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,1,128,1,float16,fp8,0,8.810304005940756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,1,128,1,fp8,fp8,0,5.701503753662109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,2,128,1,float16,float16,0,8.594869613647461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,64,128,1,float16,float16,0,8.54586664835612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,2,128,1,float16,fp8,0,8.662026723225912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,64,128,1,float16,fp8,0,8.492506663004557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,64,128,1,fp8,fp8,0,5.655237197875977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,2,128,1,fp8,fp8,0,5.67079480489095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,4,128,1,float16,float16,0,8.709722518920898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,4,128,1,float16,fp8,0,8.593648274739584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,8,128,1,float16,fp8,0,8.649311701456705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,4,128,1,fp8,fp8,0,5.646106719970703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,8,128,1,float16,float16,0,8.700613021850586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,64,8,128,1,fp8,fp8,0,5.689866383870442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,1,128,1,float16,float16,0,39.61444854736328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,1,128,1,float16,fp8,0,38.72709401448568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,1,128,1,fp8,fp8,0,25.95033009847005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,2,128,1,float16,float16,0,38.47717793782552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,2,128,1,fp8,fp8,0,25.752042134602863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,4,128,1,float16,float16,0,38.341573079427086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,2,128,1,float16,fp8,0,38.53522237141927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,4,128,1,float16,fp8,0,39.66891733805338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,4,128,1,fp8,fp8,0,26.0205815633138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,8,128,1,float16,float16,0,39.209190368652344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,8,128,1,float16,fp8,0,38.54612731933594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,64,8,128,1,fp8,fp8,0,26.052714029947918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,1,128,1,float16,float16,0,19.66002146402995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,64,128,1,float16,float16,0,19.232789357503254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,64,128,1,float16,fp8,0,19.285125732421875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,1,128,1,float16,fp8,0,19.515008290608723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,1,128,1,fp8,fp8,0,12.987327575683594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,2,128,1,float16,fp8,0,19.36190923055013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,2,128,1,float16,float16,0,19.25237274169922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,2,128,1,fp8,fp8,0,12.812496185302734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,64,128,1,fp8,fp8,0,13.650559743245443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,4,128,1,float16,fp8,0,19.245418548583984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,4,128,1,fp8,fp8,0,12.991077423095703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,4,128,1,float16,float16,0,19.35809071858724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,1,128,1,float16,float16,0,9.710447947184244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,8,128,1,float16,float16,0,19.619253794352215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,8,128,1,fp8,fp8,0,12.955851236979166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,64,8,128,1,float16,fp8,0,19.643370310465496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,64,128,1,float16,float16,0,10.039642969767252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,1,128,1,float16,fp8,0,9.581439971923828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,64,128,1,fp8,fp8,0,6.910645167032878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,1,128,1,fp8,fp8,0,6.451525370279948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,64,128,1,float16,fp8,0,9.869754791259766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,2,128,1,float16,float16,0,9.735685348510742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,2,128,1,float16,fp8,0,9.829658508300781
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,2,128,1,fp8,fp8,0,6.571285247802734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,4,128,1,float16,fp8,0,9.698687871297201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,4,128,1,fp8,fp8,0,6.619173049926758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,4,128,1,float16,float16,0,9.88160514831543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,8,128,1,float16,float16,0,9.999445597330729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,8,128,1,float16,fp8,0,9.66550381978353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,64,8,128,1,fp8,fp8,0,6.726997375488281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,1,128,1,float16,float16,0,4.855130513509114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,1,128,1,float16,fp8,0,4.924703915913899
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,1,128,1,fp8,fp8,0,3.3987948099772134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,2,128,1,float16,float16,0,4.812282562255859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,64,128,1,float16,float16,0,5.085957209269206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,64,128,1,float16,fp8,0,4.927066802978516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,64,128,1,fp8,fp8,0,3.4534079233805337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,2,128,1,fp8,fp8,0,3.4024747212727866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,2,128,1,float16,fp8,0,4.85426139831543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,4,128,1,float16,float16,0,4.938693364461263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,4,128,1,fp8,fp8,0,3.412848154703776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,4,128,1,float16,fp8,0,4.936938603719075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,8,128,1,float16,float16,0,4.809701283772786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,8,128,1,float16,fp8,0,4.8155466715494795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,64,8,128,1,fp8,fp8,0,3.4124320348103843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,1,128,1,float16,fp8,0,27.028165181477863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,1,128,1,float16,float16,0,27.342793782552082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,1,128,1,fp8,fp8,0,19.051637013753254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,2,128,1,float16,fp8,0,27.198646545410156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,2,128,1,fp8,fp8,0,18.668720245361328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,2,128,1,float16,float16,0,27.75067647298177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,4,128,1,float16,float16,0,27.66552480061849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,4,128,1,float16,fp8,0,27.094256083170574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,4,128,1,fp8,fp8,0,18.3636474609375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,8,128,1,float16,float16,0,28.223317464192707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,8,128,1,fp8,fp8,0,18.630879720052082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,64,8,128,1,float16,fp8,0,27.15637969970703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,64,128,1,float16,float16,0,13.997263590494791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,64,128,1,float16,fp8,0,13.999008178710938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,1,128,1,float16,fp8,0,14.000389099121094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,1,128,1,float16,float16,0,13.7041384379069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,1,128,1,fp8,fp8,0,9.335770924886068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,2,128,1,float16,float16,0,13.855674743652344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,2,128,1,float16,fp8,0,13.7303835550944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,2,128,1,fp8,fp8,0,9.31066640218099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,4,128,1,float16,float16,0,14.093252817789713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,4,128,1,fp8,fp8,0,9.340976079305014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,4,128,1,float16,fp8,0,13.58996327718099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,8,128,1,float16,float16,0,13.708714803059896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,64,128,1,fp8,fp8,0,9.573216120402018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,1,128,1,float16,float16,0,6.978111902872722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,1,128,1,float16,fp8,0,6.773866653442383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,8,128,1,float16,fp8,0,13.733690897623697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,64,8,128,1,fp8,fp8,0,9.342554728190104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,1,128,1,fp8,fp8,0,4.643738746643066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,64,128,1,float16,float16,0,6.920890808105469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,64,128,1,float16,fp8,0,7.074629465738933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,2,128,1,float16,float16,0,6.854709625244141
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,2,128,1,float16,fp8,0,6.798997243245442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,2,128,1,fp8,fp8,0,4.650143941243489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,64,128,1,fp8,fp8,0,4.7734880447387695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,4,128,1,float16,float16,0,6.975840250651042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,4,128,1,float16,fp8,0,7.092704137166341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,4,128,1,fp8,fp8,0,4.667269388834636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,8,128,1,float16,float16,0,6.916912078857422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,8,128,1,float16,fp8,0,6.872133255004883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,64,8,128,1,fp8,fp8,0,4.667130788167317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,1,128,1,float16,float16,0,3.5293172200520835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,1,128,1,float16,fp8,0,3.4334932963053384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,64,128,1,float16,float16,0,3.446165402730306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,1,128,1,fp8,fp8,0,2.4969919522603354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,2,128,1,float16,float16,0,3.4781599044799805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,2,128,1,float16,fp8,0,3.4927520751953125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,64,128,1,float16,fp8,0,3.430255889892578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,64,128,1,fp8,fp8,0,2.5549866358439126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,2,128,1,fp8,fp8,0,2.499237378438314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,4,128,1,float16,fp8,0,3.478954633076986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,4,128,1,fp8,fp8,0,2.503647963205973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,4,128,1,float16,float16,0,3.478879928588867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,8,128,1,float16,float16,0,3.518517176310221
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,8,128,1,float16,fp8,0,3.428842544555664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,64,8,128,1,fp8,fp8,0,2.509237289428711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,1,128,1,float16,float16,0,37.0931142171224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,1,128,1,fp8,fp8,0,24.712603251139324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,2,128,1,fp8,fp8,0,24.857716878255207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,1,128,1,float16,fp8,0,37.25770568847656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,2,128,1,float16,fp8,0,37.537071228027344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,4,128,1,float16,float16,0,38.87890625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,2,128,1,float16,float16,0,37.54913075764974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,4,128,1,float16,fp8,0,37.166351318359375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,8,128,1,fp8,fp8,0,24.61041514078776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,8,128,1,float16,float16,0,38.20584615071615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,4,128,1,fp8,fp8,0,24.86322021484375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,64,8,128,1,float16,fp8,0,37.991129557291664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,64,128,1,float16,float16,0,19.002389272054035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,64,128,1,float16,fp8,0,18.100048065185547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,1,128,1,float16,fp8,0,17.861365000406902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,1,128,1,float16,float16,0,18.146442413330078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,1,128,1,fp8,fp8,0,12.435914357503256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,2,128,1,float16,float16,0,18.028746287027996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,2,128,1,float16,fp8,0,18.40760040283203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,2,128,1,fp8,fp8,0,12.311152140299479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,4,128,1,float16,float16,0,18.094256083170574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,4,128,1,float16,fp8,0,18.166906992594402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,4,128,1,fp8,fp8,0,12.402416229248047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,8,128,1,float16,float16,0,18.25222396850586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,64,128,1,fp8,fp8,0,12.721722920735678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,8,128,1,float16,fp8,0,18.320560455322266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,1,128,1,float16,float16,0,9.126384099324545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,64,8,128,1,fp8,fp8,0,12.29422378540039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,1,128,1,float16,fp8,0,8.85592524210612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,1,128,1,fp8,fp8,0,6.296309153238933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,64,128,1,float16,fp8,0,9.208208084106445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,64,128,1,float16,float16,0,9.281007766723633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,64,128,1,fp8,fp8,0,6.536186854044597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,2,128,1,float16,float16,0,9.051626841227213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,2,128,1,fp8,fp8,0,6.176176071166992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,2,128,1,float16,fp8,0,9.030501047770182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,4,128,1,float16,float16,0,9.15219751993815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,4,128,1,float16,fp8,0,9.031551996866861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,4,128,1,fp8,fp8,0,6.131551742553711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,8,128,1,float16,float16,0,9.243317286173502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,8,128,1,float16,fp8,0,8.902458826700846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,64,8,128,1,fp8,fp8,0,6.1242936452229815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,1,128,1,float16,float16,0,4.567690531412761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,64,128,1,float16,float16,0,4.5591786702473955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,1,128,1,float16,fp8,0,4.529093424479167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,1,128,1,fp8,fp8,0,3.1726719538370767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,2,128,1,float16,float16,0,4.38262939453125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,2,128,1,float16,fp8,0,4.391749382019043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,64,128,1,float16,fp8,0,4.522778511047363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,2,128,1,fp8,fp8,0,3.1724640528361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,64,128,1,fp8,fp8,0,3.2825492223103843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,4,128,1,float16,float16,0,4.525477409362793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,4,128,1,float16,fp8,0,4.346703847249349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,4,128,1,fp8,fp8,0,3.178346633911133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,8,128,1,float16,float16,0,4.561445236206055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,8,128,1,float16,fp8,0,4.5157012939453125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,64,8,128,1,fp8,fp8,0,3.1927359898885093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,64,128,1,float16,float16,0,2.35044797261556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,1,128,1,float16,float16,0,2.3381919860839844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,1,128,1,float16,fp8,0,2.3074986139933267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,1,128,1,fp8,fp8,0,1.733045260111491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,2,128,1,float16,float16,0,2.3392319679260254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,2,128,1,float16,fp8,0,2.310544013977051
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,64,128,1,float16,fp8,0,2.3426666259765625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,64,128,1,fp8,fp8,0,1.77675199508667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,2,128,1,fp8,fp8,0,1.7295039494832356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,4,128,1,float16,float16,0,2.3447680473327637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,4,128,1,float16,fp8,0,2.3639465967814126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,4,128,1,fp8,fp8,0,1.7302026748657227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,8,128,1,float16,float16,0,2.3494505882263184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,8,128,1,float16,fp8,0,2.3312692642211914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,64,8,128,1,fp8,fp8,0,1.7349440256754558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,1,128,1,float16,fp8,0,20.977855682373047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,1,128,1,fp8,fp8,0,15.192501068115234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,1,128,1,float16,float16,0,21.204447428385418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,2,128,1,float16,float16,0,21.124293009440105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,2,128,1,float16,fp8,0,21.084944407145183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,2,128,1,fp8,fp8,0,14.853375752766928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,4,128,1,float16,float16,0,21.510836283365887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,4,128,1,float16,fp8,0,21.364939371744793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,4,128,1,fp8,fp8,0,15.229456583658854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,8,128,1,float16,float16,0,21.370704650878906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,8,128,1,fp8,fp8,0,15.06390380859375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,64,8,128,1,float16,fp8,0,20.874549865722656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,64,128,1,float16,float16,0,10.963141123453775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,1,128,1,float16,float16,0,10.81780751546224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,64,128,1,float16,fp8,0,10.897445678710938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,1,128,1,fp8,fp8,0,7.511434555053711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,1,128,1,float16,fp8,0,10.386202494303385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,2,128,1,float16,float16,0,10.688298543294271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,2,128,1,fp8,fp8,0,7.730560302734375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,2,128,1,float16,fp8,0,10.650735855102539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,4,128,1,float16,float16,0,10.654912312825521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,4,128,1,float16,fp8,0,10.761845906575521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,4,128,1,fp8,fp8,0,7.386346817016602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,64,128,1,fp8,fp8,0,7.752378463745117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,8,128,1,float16,float16,0,10.6070187886556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,1,128,1,float16,float16,0,5.076517422993978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,8,128,1,float16,fp8,0,10.887940724690756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,64,8,128,1,fp8,fp8,0,7.5127410888671875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,1,128,1,float16,fp8,0,5.091381390889485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,64,128,1,float16,float16,0,5.394538879394531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,1,128,1,fp8,fp8,0,3.74071470896403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,2,128,1,float16,float16,0,5.358234405517578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,64,128,1,float16,fp8,0,5.436570485432942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,2,128,1,float16,fp8,0,5.079279899597168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,64,128,1,fp8,fp8,0,3.9468425114949546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,2,128,1,fp8,fp8,0,3.747722625732422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,4,128,1,float16,float16,0,5.37933349609375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,4,128,1,float16,fp8,0,5.130688031514485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,4,128,1,fp8,fp8,0,3.756058692932129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,8,128,1,float16,float16,0,5.287887891133626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,8,128,1,float16,fp8,0,5.057792027791341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,64,8,128,1,fp8,fp8,0,3.773263931274414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,1,128,1,float16,float16,0,2.688607851664225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,1,128,1,float16,fp8,0,2.6062186559041343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,1,128,1,fp8,fp8,0,1.9730772972106934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,64,128,1,float16,float16,0,2.684746742248535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,2,128,1,float16,float16,0,2.6462880770365396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,64,128,1,fp8,fp8,0,2.0676053365071616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,64,128,1,float16,fp8,0,2.67521603902181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,2,128,1,float16,fp8,0,2.6650986671447754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,2,128,1,fp8,fp8,0,1.9743040402730305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,4,128,1,float16,float16,0,2.6612319946289062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,4,128,1,float16,fp8,0,2.6084373792012534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,4,128,1,fp8,fp8,0,1.9766613642374675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,8,128,1,float16,float16,0,2.656895955403646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,8,128,1,fp8,fp8,0,1.9888639450073242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,64,8,128,1,float16,fp8,0,2.6233386993408203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,1,128,1,float16,float16,0,1.4397974014282227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,64,128,1,float16,float16,0,1.4517226219177246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,1,128,1,float16,fp8,0,1.4139413833618164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,64,128,1,float16,fp8,0,1.4400746027628581
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,1,128,1,fp8,fp8,0,1.0956213474273682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,64,128,1,fp8,fp8,0,1.1359039942423503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,2,128,1,float16,float16,0,1.4444106419881184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,2,128,1,float16,fp8,0,1.413823922475179
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,2,128,1,fp8,fp8,0,1.0970133145650227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,4,128,1,float16,float16,0,1.4424586296081543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,4,128,1,float16,fp8,0,1.4206666946411133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,4,128,1,fp8,fp8,0,1.0982133547465007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,8,128,1,float16,fp8,0,1.420954704284668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,8,128,1,fp8,fp8,0,1.101482629776001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,64,8,128,1,float16,float16,0,1.4432479540507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,1,128,1,float16,float16,0,21.984148661295574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,1,128,1,float16,fp8,0,21.760836283365887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,1,128,1,fp8,fp8,0,15.001861572265625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,2,128,1,fp8,fp8,0,14.971728006998697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,2,128,1,float16,fp8,0,21.585866292317707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,2,128,1,float16,float16,0,22.30511983235677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,4,128,1,float16,float16,0,21.934539794921875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,4,128,1,float16,fp8,0,21.836522420247395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,4,128,1,fp8,fp8,0,15.008490244547525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,8,128,1,float16,float16,0,22.63511912027995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,8,128,1,float16,fp8,0,22.08556365966797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,64,8,128,1,fp8,fp8,0,15.049242655436197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,1,128,1,float16,float16,0,10.278085072835287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,64,128,1,float16,float16,0,10.884756724039713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,1,128,1,float16,fp8,0,10.295845031738281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,1,128,1,fp8,fp8,0,7.484810511271159
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,2,128,1,float16,float16,0,10.346138636271158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,64,128,1,float16,fp8,0,10.721856435139975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,2,128,1,float16,fp8,0,10.172213236490885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,2,128,1,fp8,fp8,0,7.617173512776692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,4,128,1,float16,fp8,0,10.385471979777018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,64,128,1,fp8,fp8,0,8.022010803222656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,4,128,1,fp8,fp8,0,7.511786778767903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,4,128,1,float16,float16,0,10.384629567464193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,8,128,1,float16,fp8,0,10.27884292602539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,1,128,1,float16,float16,0,5.115647951761882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,8,128,1,float16,float16,0,10.300021489461264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,64,8,128,1,fp8,fp8,0,7.720021565755208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,1,128,1,float16,fp8,0,4.899706522623698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,1,128,1,fp8,fp8,0,3.7718238830566406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,64,128,1,float16,fp8,0,5.304906527201335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,64,128,1,float16,float16,0,5.270821253458659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,64,128,1,fp8,fp8,0,4.056586583455403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,2,128,1,float16,fp8,0,5.040458679199219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,2,128,1,float16,float16,0,5.061029434204102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,2,128,1,fp8,fp8,0,3.7711785634358725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,4,128,1,float16,fp8,0,4.948479970296224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,4,128,1,fp8,fp8,0,3.78436279296875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,4,128,1,float16,float16,0,5.094714800516765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,8,128,1,float16,fp8,0,5.188655853271484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,8,128,1,float16,float16,0,5.197696050008138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,64,8,128,1,fp8,fp8,0,3.8084373474121094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,1,128,1,float16,float16,0,2.535162607828776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,64,128,1,float16,float16,0,2.637392044067383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,1,128,1,float16,fp8,0,2.5184799830118814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,1,128,1,fp8,fp8,0,1.952837308247884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,2,128,1,float16,float16,0,2.5279359817504883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,64,128,1,float16,fp8,0,2.600714683532715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,2,128,1,fp8,fp8,0,1.955077330271403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,64,128,1,fp8,fp8,0,2.103306611378988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,2,128,1,float16,fp8,0,2.494277318318685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,4,128,1,float16,float16,0,2.572826703389486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,4,128,1,float16,fp8,0,2.5070346196492515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,8,128,1,float16,fp8,0,2.497887929280599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,8,128,1,float16,float16,0,2.543813387552897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,4,128,1,fp8,fp8,0,1.961045265197754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,64,8,128,1,fp8,fp8,0,1.9707040786743164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,64,128,1,float16,fp8,0,1.383786678314209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,64,128,1,float16,float16,0,1.3898132642110188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,64,128,1,fp8,fp8,0,1.119066635767619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,1,128,1,float16,float16,0,1.3434133529663086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,1,128,1,float16,fp8,0,1.3142879803975422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,2,128,1,float16,float16,0,1.3424266179402669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,1,128,1,fp8,fp8,0,1.04966934521993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,2,128,1,float16,fp8,0,1.3307680288950603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,4,128,1,float16,float16,0,1.3502400716145833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,4,128,1,float16,fp8,0,1.3189067045847576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,4,128,1,fp8,fp8,0,1.053877353668213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,2,128,1,fp8,fp8,0,1.0507573286692302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,8,128,1,float16,float16,0,1.3477546374003093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,8,128,1,fp8,fp8,0,1.0586826801300049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,1,128,1,float16,fp8,0,0.7347520192464193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,64,8,128,1,float16,fp8,0,1.3228267033894856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,64,128,1,float16,float16,0,0.7641706466674805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,1,128,1,float16,float16,0,0.7543946901957194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,64,128,1,float16,fp8,0,0.7511093616485596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,1,128,1,fp8,fp8,0,0.6018720070521036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,64,128,1,fp8,fp8,0,0.6316373348236084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,2,128,1,float16,float16,0,0.7551413377126058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,2,128,1,fp8,fp8,0,0.6047466595967611
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,2,128,1,float16,fp8,0,0.7377386887868246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,4,128,1,fp8,fp8,0,0.6037493149439493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,4,128,1,float16,fp8,0,0.738912026087443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,4,128,1,float16,float16,0,0.7570666472117106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,8,128,1,float16,float16,0,0.7571626504262289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,8,128,1,float16,fp8,0,0.7389919757843018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,64,8,128,1,fp8,fp8,0,0.6098666588465372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,1,128,1,float16,float16,0,12.50002670288086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,1,128,1,float16,fp8,0,12.706186930338541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,2,128,1,fp8,fp8,0,9.505664189656576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,2,128,1,float16,float16,0,12.75158437093099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,1,128,1,fp8,fp8,0,9.496224085489908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,4,128,1,float16,float16,0,12.800501505533854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,2,128,1,float16,fp8,0,12.598047892252604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,4,128,1,float16,fp8,0,12.452096303304037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,4,128,1,fp8,fp8,0,9.605557123819986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,8,128,1,float16,float16,0,12.719056447347006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,8,128,1,fp8,fp8,0,9.589967727661133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,64,8,128,1,float16,fp8,0,12.630602518717447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,64,128,1,float16,fp8,0,6.53981335957845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,64,128,1,float16,float16,0,6.752255757649739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,1,128,1,float16,float16,0,6.246842702229817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,1,128,1,float16,fp8,0,6.16270383199056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,1,128,1,fp8,fp8,0,4.769205411275228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,2,128,1,float16,float16,0,6.144970575968425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,2,128,1,float16,fp8,0,6.125989278157552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,2,128,1,fp8,fp8,0,4.787450790405273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,4,128,1,float16,float16,0,6.157360076904297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,4,128,1,fp8,fp8,0,4.791647911071777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,4,128,1,float16,fp8,0,6.0649064381917315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,64,128,1,fp8,fp8,0,5.168490727742513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,8,128,1,float16,float16,0,6.189530690511067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,8,128,1,float16,fp8,0,6.310960133870442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,64,8,128,1,fp8,fp8,0,4.826479911804199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,1,128,1,float16,float16,0,3.0402825673421225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,1,128,1,fp8,fp8,0,2.416111946105957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,64,128,1,float16,float16,0,3.3309332529703775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,64,128,1,float16,fp8,0,3.1951093673706055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,2,128,1,float16,float16,0,3.047546704610189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,1,128,1,float16,fp8,0,2.982405344645182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,64,128,1,fp8,fp8,0,2.639472007751465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,2,128,1,float16,fp8,0,2.9788853327433267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,4,128,1,float16,fp8,0,2.984287897745768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,4,128,1,float16,float16,0,3.091808001200358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,4,128,1,fp8,fp8,0,2.4296372731526694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,2,128,1,fp8,fp8,0,2.419109344482422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,8,128,1,fp8,fp8,0,2.4429920514424643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,8,128,1,float16,fp8,0,3.0100959142049155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,1,128,1,float16,float16,0,1.5816960334777832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,64,8,128,1,float16,float16,0,3.110191980997721
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,1,128,1,float16,fp8,0,1.546778678894043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,1,128,1,fp8,fp8,0,1.2643893559773762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,2,128,1,float16,float16,0,1.58461332321167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,2,128,1,float16,fp8,0,1.5509653091430664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,64,128,1,float16,float16,0,1.6669227282206218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,64,128,1,float16,fp8,0,1.6683519681294758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,64,128,1,fp8,fp8,0,1.3843894004821777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,2,128,1,fp8,fp8,0,1.2670880158742268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,4,128,1,float16,float16,0,1.5841919581095378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,4,128,1,float16,fp8,0,1.5551786422729492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,4,128,1,fp8,fp8,0,1.2718186378479004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,8,128,1,float16,float16,0,1.5906826655069988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,8,128,1,float16,fp8,0,1.5759414037068684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,64,8,128,1,fp8,fp8,0,1.2804426352183025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,64,128,1,float16,fp8,0,0.8654293219248453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,1,128,1,float16,float16,0,0.852405309677124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,64,128,1,float16,float16,0,0.8792426586151123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,1,128,1,float16,fp8,0,0.8315893014272054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,1,128,1,fp8,fp8,0,0.6903946399688721
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,2,128,1,float16,float16,0,0.8577866554260254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,64,128,1,fp8,fp8,0,0.7487946351369222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,2,128,1,float16,fp8,0,0.832751989364624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,2,128,1,fp8,fp8,0,0.6911946932474772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,4,128,1,float16,float16,0,0.8542079925537109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,4,128,1,fp8,fp8,0,0.6937386989593506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,4,128,1,float16,fp8,0,0.8353172938028971
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,8,128,1,float16,float16,0,0.8618346850077311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,8,128,1,float16,fp8,0,0.8390826384226481
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,64,8,128,1,fp8,fp8,0,0.696458657582601
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,1,128,1,float16,float16,0,0.49029866854349774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,64,128,1,float16,float16,0,0.500490665435791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,2,128,1,float16,float16,0,0.49186134338378906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,1,128,1,fp8,fp8,0,0.38974400361378986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,64,128,1,float16,fp8,0,0.4910453160603841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,1,128,1,float16,fp8,0,0.4792160193125407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,64,128,1,fp8,fp8,0,0.42153600851694745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,2,128,1,float16,fp8,0,0.47892268498738605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,2,128,1,fp8,fp8,0,0.3887360095977783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,4,128,1,float16,float16,0,0.4912480115890503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,4,128,1,float16,fp8,0,0.48131199677785236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,8,128,1,float16,float16,0,0.493887980779012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,4,128,1,fp8,fp8,0,0.3890560070673625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,8,128,1,fp8,fp8,0,0.39202133814493817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,64,8,128,1,float16,fp8,0,0.48206933339436847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,1,128,1,float16,fp8,0,13.975210825602213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,2,128,1,fp8,fp8,0,10.580554962158203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,2,128,1,float16,fp8,0,14.108912150065104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,1,128,1,float16,float16,0,14.5675417582194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,1,128,1,fp8,fp8,0,10.54747200012207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,2,128,1,float16,float16,0,14.679803212483725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,4,128,1,float16,float16,0,14.96890131632487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,4,128,1,float16,fp8,0,14.195013682047525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,4,128,1,fp8,fp8,0,10.59165891011556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,8,128,1,fp8,fp8,0,10.61246426900228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,8,128,1,float16,float16,0,14.968154907226562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,64,128,1,float16,fp8,0,6.935210545857747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,1,128,1,float16,float16,0,6.413232167561849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,64,8,128,1,float16,fp8,0,14.69207509358724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,64,128,1,float16,float16,0,6.9745229085286455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,1,128,1,float16,fp8,0,6.322751998901367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,1,128,1,fp8,fp8,0,5.222757339477539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,2,128,1,float16,float16,0,6.571461359659831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,2,128,1,float16,fp8,0,6.146202723185222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,2,128,1,fp8,fp8,0,5.228303909301758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,4,128,1,float16,fp8,0,6.274864196777344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,4,128,1,float16,float16,0,6.388261159261067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,64,128,1,fp8,fp8,0,5.75816027323405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,4,128,1,fp8,fp8,0,5.249967892964681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,8,128,1,float16,float16,0,6.549296061197917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,1,128,1,float16,float16,0,3.1241172154744468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,8,128,1,float16,fp8,0,6.271376291910808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,64,8,128,1,fp8,fp8,0,5.285781224568685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,1,128,1,float16,fp8,0,3.076266606648763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,1,128,1,fp8,fp8,0,2.608293374379476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,64,128,1,float16,float16,0,3.455103874206543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,64,128,1,float16,fp8,0,3.381162643432617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,2,128,1,float16,float16,0,3.1274665196736655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,64,128,1,fp8,fp8,0,2.9139254887898765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,2,128,1,float16,fp8,0,3.0474828084309897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,2,128,1,fp8,fp8,0,2.6147573788960776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,4,128,1,float16,float16,0,3.136741320292155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,4,128,1,float16,fp8,0,3.059253374735514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,4,128,1,fp8,fp8,0,2.6238452593485513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,8,128,1,float16,float16,0,3.1537599563598633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,8,128,1,float16,fp8,0,3.069808006286621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,64,8,128,1,fp8,fp8,0,2.6453706423441568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,1,128,1,float16,float16,0,1.6042985916137695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,1,128,1,float16,fp8,0,1.5606613159179688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,1,128,1,fp8,fp8,0,1.3421707153320312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,64,128,1,float16,float16,0,1.7775786717732747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,2,128,1,float16,float16,0,1.607434590657552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,64,128,1,fp8,fp8,0,1.5007786750793457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,2,128,1,float16,fp8,0,1.5648800532023113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,2,128,1,fp8,fp8,0,1.3459946314493816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,64,128,1,float16,fp8,0,1.711498737335205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,4,128,1,float16,float16,0,1.6115840276082356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,4,128,1,float16,fp8,0,1.5806719462076824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,4,128,1,fp8,fp8,0,1.3518452644348145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,8,128,1,float16,float16,0,1.621946652730306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,8,128,1,float16,fp8,0,1.5784533818562825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,64,8,128,1,fp8,fp8,0,1.3642932573954265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,1,128,1,float16,fp8,0,0.8242133458455404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,1,128,1,fp8,fp8,0,0.7134186426798502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,1,128,1,float16,float16,0,0.8442186514536539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,64,128,1,float16,fp8,0,0.8851733207702637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,64,128,1,fp8,fp8,0,0.796282688776652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,2,128,1,float16,float16,0,0.8485120137532552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,64,128,1,float16,float16,0,0.9070560137430826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,2,128,1,float16,fp8,0,0.82696533203125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,2,128,1,fp8,fp8,0,0.7151892979939779
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,4,128,1,float16,fp8,0,0.8276800314585367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,4,128,1,float16,float16,0,0.8497653007507324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,4,128,1,fp8,fp8,0,0.7189493179321289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,8,128,1,float16,float16,0,0.8550026416778564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,1,128,1,float16,float16,0,0.46834667523701984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,8,128,1,fp8,fp8,0,0.7247146765391032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,1,128,1,float16,fp8,0,0.4545493523279826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,64,8,128,1,float16,fp8,0,0.832810640335083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,64,128,1,float16,float16,0,0.4904373486836751
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,64,128,1,float16,fp8,0,0.4801386594772339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,64,128,1,fp8,fp8,0,0.4364000161488851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,1,128,1,fp8,fp8,0,0.39948801199595135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,2,128,1,float16,float16,0,0.4700106779734294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,2,128,1,float16,fp8,0,0.4556106726328532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,2,128,1,fp8,fp8,0,0.39922134081522626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,4,128,1,float16,float16,0,0.4710719982783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,4,128,1,float16,fp8,0,0.4562346537907918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,8,128,1,float16,float16,0,0.47304534912109375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,4,128,1,fp8,fp8,0,0.40005334218343097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,8,128,1,float16,fp8,0,0.4594293435414632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,64,8,128,1,fp8,fp8,0,0.4040000041325887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,64,128,1,float16,float16,0,0.2887733379999797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,64,128,1,float16,fp8,0,0.282640000184377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,64,128,1,fp8,fp8,0,0.2512320081392924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,1,128,1,float16,float16,0,0.27967466910680133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,1,128,1,fp8,fp8,0,0.22949866453806558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,1,128,1,float16,fp8,0,0.2707200050354004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,2,128,1,float16,float16,0,0.2795520027478536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,2,128,1,fp8,fp8,0,0.2295786738395691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,4,128,1,float16,float16,0,0.2807413339614868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,2,128,1,float16,fp8,0,0.27102933327356976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,4,128,1,float16,fp8,0,0.27376000086466473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,4,128,1,fp8,fp8,0,0.23061333100001016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,8,128,1,float16,fp8,0,0.2741706569989522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,8,128,1,float16,float16,0,0.28196799755096436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,64,8,128,1,fp8,fp8,0,0.23214934269587198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,1,128,1,fp8,fp8,0,7.06931749979655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,1,128,1,float16,float16,0,8.338144302368164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,2,128,1,float16,fp8,0,7.978928248087565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,2,128,1,float16,float16,0,8.355887730916342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,2,128,1,fp8,fp8,0,7.077285130818685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,1,128,1,float16,fp8,0,8.027941385904947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,4,128,1,float16,float16,0,8.3765869140625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,4,128,1,float16,fp8,0,8.133562723795572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,4,128,1,fp8,fp8,0,7.087605158487956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,8,128,1,float16,float16,0,8.455696105957031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,8,128,1,fp8,fp8,0,7.117504119873047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,64,8,128,1,float16,fp8,0,8.176736195882162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,1,128,1,float16,float16,0,4.052154541015625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,64,128,1,float16,fp8,0,4.440512021382649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,64,128,1,float16,float16,0,4.533445358276367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,1,128,1,float16,fp8,0,3.900735855102539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,2,128,1,float16,float16,0,4.065594673156738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,1,128,1,fp8,fp8,0,3.4822400410970054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,2,128,1,float16,fp8,0,3.906613349914551
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,2,128,1,fp8,fp8,0,3.492442766825358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,4,128,1,float16,fp8,0,3.918421427408854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,4,128,1,float16,float16,0,4.0734453201293945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,64,128,1,fp8,fp8,0,3.9269545873006186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,4,128,1,fp8,fp8,0,3.5089918772379556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,8,128,1,float16,fp8,0,3.942837397257487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,8,128,1,float16,float16,0,4.127909342447917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,1,128,1,float16,float16,0,2.0250293413798013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,64,8,128,1,fp8,fp8,0,3.5418132146199546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,1,128,1,float16,fp8,0,1.963274637858073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,1,128,1,fp8,fp8,0,1.7504693667093914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,2,128,1,float16,float16,0,2.029354731241862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,2,128,1,float16,fp8,0,1.9696745872497559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,64,128,1,float16,float16,0,2.27948792775472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,64,128,1,fp8,fp8,0,1.9944480260213215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,64,128,1,float16,fp8,0,2.216357390085856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,2,128,1,fp8,fp8,0,1.7551892598470051
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,4,128,1,float16,fp8,0,1.9737812678019206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,4,128,1,float16,float16,0,2.0341173807779946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,4,128,1,fp8,fp8,0,1.765109380086263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,8,128,1,float16,float16,0,2.046053409576416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,8,128,1,float16,fp8,0,1.9995360374450684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,1,128,1,float16,float16,0,1.0501920382181804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,64,8,128,1,fp8,fp8,0,1.7793599764506023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,1,128,1,float16,fp8,0,1.0160533587137859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,1,128,1,fp8,fp8,0,0.9101920127868652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,64,128,1,float16,float16,0,1.1608213583628337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,2,128,1,float16,float16,0,1.0525866349538167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,2,128,1,float16,fp8,0,1.0189759731292725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,64,128,1,float16,fp8,0,1.133082628250122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,2,128,1,fp8,fp8,0,0.9139839808146158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,4,128,1,float16,float16,0,1.056320031483968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,64,128,1,fp8,fp8,0,1.0333013534545898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,4,128,1,float16,fp8,0,1.0218719641367595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,4,128,1,fp8,fp8,0,0.916815996170044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,8,128,1,float16,fp8,0,1.0299466451009114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,8,128,1,float16,float16,0,1.058853308359782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,64,8,128,1,fp8,fp8,0,0.9266026814778646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,64,128,1,float16,float16,0,0.6118880112965902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,1,128,1,float16,float16,0,0.560975988705953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,1,128,1,float16,fp8,0,0.5431786775588989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,2,128,1,float16,float16,0,0.5614826679229736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,1,128,1,fp8,fp8,0,0.49000000953674316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,64,128,1,fp8,fp8,0,0.5534240007400513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,64,128,1,float16,fp8,0,0.5927573442459106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,2,128,1,float16,fp8,0,0.5454080104827881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,2,128,1,fp8,fp8,0,0.49127999941507977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,4,128,1,float16,float16,0,0.5635413328806559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,4,128,1,float16,fp8,0,0.5462293227513632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,4,128,1,fp8,fp8,0,0.4930826822916667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,8,128,1,float16,float16,0,0.5650240182876587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,8,128,1,fp8,fp8,0,0.4970133304595947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,64,8,128,1,float16,fp8,0,0.5498026609420776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,64,128,1,float16,float16,0,0.336026668548584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,1,128,1,float16,float16,0,0.31682666142781574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,1,128,1,float16,fp8,0,0.3052106698354085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,64,128,1,float16,fp8,0,0.32891732454299927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,1,128,1,fp8,fp8,0,0.2688746651013692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,64,128,1,fp8,fp8,0,0.3136639992396037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,2,128,1,float16,fp8,0,0.30799466371536255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,2,128,1,float16,float16,0,0.31677865982055664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,2,128,1,fp8,fp8,0,0.2704373399416606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,4,128,1,float16,float16,0,0.31752532720565796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,4,128,1,float16,fp8,0,0.30853333075841266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,4,128,1,fp8,fp8,0,0.2706293265024821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,8,128,1,float16,float16,0,0.3193173408508301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,8,128,1,float16,fp8,0,0.31085866689682007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,64,8,128,1,fp8,fp8,0,0.27245332797368366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,64,128,1,float16,fp8,0,0.19746132691701254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,64,128,1,float16,float16,0,0.2019360065460205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,1,128,1,float16,float16,0,0.18196266889572144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,1,128,1,float16,fp8,0,0.17580799261728922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,64,128,1,fp8,fp8,0,0.17802667617797852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,1,128,1,fp8,fp8,0,0.16386666893959045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,2,128,1,float16,float16,0,0.18228799104690552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,2,128,1,float16,fp8,0,0.17510400215784708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,2,128,1,fp8,fp8,0,0.16410666704177856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,4,128,1,float16,float16,0,0.18252267440160116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,4,128,1,float16,fp8,0,0.17637866735458374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,4,128,1,fp8,fp8,0,0.1642400026321411
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,8,128,1,float16,float16,0,0.18361600240071616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,8,128,1,float16,fp8,0,0.17695999145507812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,64,8,128,1,fp8,fp8,0,0.16621333360671997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,1,128,1,fp8,fp8,0,6.92851193745931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,1,128,1,float16,fp8,0,9.87779172261556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,1,128,1,float16,float16,0,9.81704012552897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,2,128,1,float16,float16,0,10.114367802937826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,2,128,1,float16,fp8,0,10.125749588012695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,4,128,1,float16,float16,0,10.043802897135416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,2,128,1,fp8,fp8,0,6.934250513712565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,4,128,1,float16,fp8,0,10.052213033040365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,4,128,1,fp8,fp8,0,7.005882898966472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,8,128,1,float16,fp8,0,10.238255818684896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,8,128,1,fp8,fp8,0,7.133290608723958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,64,8,128,1,float16,float16,0,10.254175821940104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,64,128,1,float16,float16,0,4.9832000732421875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,1,128,1,float16,float16,0,3.8015572230021157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,1,128,1,float16,fp8,0,3.8257226943969727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,1,128,1,fp8,fp8,0,3.4236532847086587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,64,128,1,float16,fp8,0,4.839136123657227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,2,128,1,float16,float16,0,3.8036588033040366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,2,128,1,float16,fp8,0,3.810672124226888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,2,128,1,fp8,fp8,0,3.44102414449056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,4,128,1,float16,float16,0,3.9317973454793296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,4,128,1,float16,fp8,0,3.951509475708008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,64,128,1,fp8,fp8,0,4.236687978108724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,8,128,1,float16,float16,0,3.9560108184814453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,4,128,1,fp8,fp8,0,3.487850824991862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,8,128,1,float16,fp8,0,4.010138511657715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,64,8,128,1,fp8,fp8,0,3.547114690144857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,1,128,1,float16,float16,0,1.9109867413838704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,1,128,1,float16,fp8,0,1.9116053581237793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,1,128,1,fp8,fp8,0,1.7151999473571777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,2,128,1,float16,float16,0,1.9196747144063313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,64,128,1,float16,float16,0,2.497706731160482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,2,128,1,float16,fp8,0,1.9181119600931804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,64,128,1,float16,fp8,0,2.417967955271403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,64,128,1,fp8,fp8,0,2.126255989074707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,2,128,1,fp8,fp8,0,1.7199093500773113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,4,128,1,float16,float16,0,1.9273866017659504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,4,128,1,float16,fp8,0,1.9418667157491047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,4,128,1,fp8,fp8,0,1.7352159818013508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,8,128,1,float16,float16,0,1.950069268544515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,1,128,1,float16,float16,0,0.9731413523356119
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,8,128,1,fp8,fp8,0,1.7710347175598145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,64,8,128,1,float16,fp8,0,1.9673919677734375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,64,128,1,float16,float16,0,1.259071985880534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,1,128,1,float16,fp8,0,0.972442626953125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,1,128,1,fp8,fp8,0,0.8444480101267496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,2,128,1,float16,float16,0,0.9760213692982992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,2,128,1,float16,fp8,0,0.9762399991353353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,64,128,1,float16,fp8,0,1.2004693349202473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,64,128,1,fp8,fp8,0,1.068991978963216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,2,128,1,fp8,fp8,0,0.8507200082143148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,4,128,1,float16,float16,0,0.9791733423868815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,4,128,1,float16,fp8,0,0.9822186628977457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,4,128,1,fp8,fp8,0,0.8771946430206299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,8,128,1,float16,float16,0,0.9910613695780436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,8,128,1,float16,fp8,0,0.9891306559244791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,64,8,128,1,fp8,fp8,0,0.894933303197225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,1,128,1,float16,float16,0,0.5004213253657023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,1,128,1,float16,fp8,0,0.501141349474589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,1,128,1,fp8,fp8,0,0.4362399975458781
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,64,128,1,float16,float16,0,0.6363946596781412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,2,128,1,float16,float16,0,0.5037226676940918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,64,128,1,fp8,fp8,0,0.5400480031967163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,64,128,1,float16,fp8,0,0.6116746664047241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,2,128,1,float16,fp8,0,0.5029280185699463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,2,128,1,fp8,fp8,0,0.43882668018341064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,4,128,1,float16,float16,0,0.5061386823654175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,4,128,1,float16,fp8,0,0.5055093367894491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,4,128,1,fp8,fp8,0,0.4424479802449544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,8,128,1,float16,float16,0,0.5117973486582438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,8,128,1,float16,fp8,0,0.5102399984995524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,64,8,128,1,fp8,fp8,0,0.44999468326568604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,1,128,1,float16,float16,0,0.2650773326555888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,64,128,1,float16,float16,0,0.33248533805211383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,64,128,1,float16,fp8,0,0.3214240074157715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,1,128,1,fp8,fp8,0,0.2320800026257833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,1,128,1,float16,fp8,0,0.2654506762822469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,64,128,1,fp8,fp8,0,0.28003732363382977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,2,128,1,float16,float16,0,0.266757329305013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,2,128,1,float16,fp8,0,0.2669813235600789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,2,128,1,fp8,fp8,0,0.2320693333943685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,4,128,1,float16,float16,0,0.26763200759887695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,4,128,1,fp8,fp8,0,0.23353066047032675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,4,128,1,float16,fp8,0,0.26744532585144043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,8,128,1,float16,fp8,0,0.2714879910151164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,8,128,1,fp8,fp8,0,0.23690134286880493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,64,8,128,1,float16,float16,0,0.2717919945716858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,64,128,1,float16,float16,0,0.1805866758028666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,1,128,1,float16,float16,0,0.1453013320763906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,1,128,1,float16,fp8,0,0.14602133631706238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,64,128,1,float16,fp8,0,0.17479999860127768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,1,128,1,fp8,fp8,0,0.12238933642705281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,64,128,1,fp8,fp8,0,0.15056533614794412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,2,128,1,float16,float16,0,0.14707733194033304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,2,128,1,float16,fp8,0,0.14707199732462564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,2,128,1,fp8,fp8,0,0.12293333808581035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,4,128,1,float16,float16,0,0.14858133594195047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,4,128,1,float16,fp8,0,0.14867200454076132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,4,128,1,fp8,fp8,0,0.12356266379356384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,8,128,1,float16,float16,0,0.15015467007954916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,8,128,1,fp8,fp8,0,0.12654933333396912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,64,8,128,1,float16,fp8,0,0.15026666720708212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,1,128,1,float16,float16,0,0.07842666904131572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,1,128,1,float16,fp8,0,0.07912533481915791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,64,128,1,fp8,fp8,0,0.084906667470932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,64,128,1,float16,float16,0,0.0999679962793986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,1,128,1,fp8,fp8,0,0.0684799998998642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,64,128,1,float16,fp8,0,0.09761599699656169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,2,128,1,float16,fp8,0,0.0792799989382426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,2,128,1,fp8,fp8,0,0.06899733344713847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,4,128,1,float16,float16,0,0.07976000010967255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,2,128,1,float16,float16,0,0.07867200175921123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,4,128,1,float16,fp8,0,0.07939733564853668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,4,128,1,fp8,fp8,0,0.0690826674302419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,8,128,1,float16,float16,0,0.08083733419577281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,8,128,1,float16,fp8,0,0.08042666812737782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,64,8,128,1,fp8,fp8,0,0.07053333520889282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,1,128,1,float16,float16,0,7.304175694783528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,2,128,1,float16,float16,0,7.636362711588542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,2,128,1,fp8,fp8,0,5.741088231404622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,4,128,1,float16,float16,0,7.664165496826172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,1,128,1,fp8,fp8,0,5.740570704142253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,1,128,1,float16,fp8,0,7.421562830607097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,2,128,1,float16,fp8,0,7.596058527628581
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,4,128,1,float16,fp8,0,7.6692854563395185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,4,128,1,fp8,fp8,0,5.835898717244466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,8,128,1,float16,float16,0,7.88968022664388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,8,128,1,float16,fp8,0,7.807167689005534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,64,8,128,1,fp8,fp8,0,5.937711715698242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,64,128,1,float16,float16,0,4.193536122639974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,1,128,1,float16,float16,0,2.931525230407715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,64,128,1,float16,fp8,0,4.020607948303223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,1,128,1,float16,fp8,0,2.9321120580037436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,1,128,1,fp8,fp8,0,2.8329814275105796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,2,128,1,float16,float16,0,2.946085294087728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,2,128,1,float16,fp8,0,2.9660746256510415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,4,128,1,float16,float16,0,3.065151850382487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,2,128,1,fp8,fp8,0,2.8386452992757163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,4,128,1,float16,fp8,0,3.0883893966674805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,4,128,1,fp8,fp8,0,2.875354766845703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,64,128,1,fp8,fp8,0,3.6512425740559897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,8,128,1,float16,float16,0,3.1539894739786782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,1,128,1,float16,float16,0,1.4845387140909831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,8,128,1,float16,fp8,0,3.1557652155558267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,64,8,128,1,fp8,fp8,0,2.9564107259114585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,1,128,1,float16,fp8,0,1.4803573290507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,64,128,1,float16,float16,0,2.101125399271647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,64,128,1,float16,fp8,0,2.020912011464437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,1,128,1,fp8,fp8,0,1.4119893709818523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,64,128,1,fp8,fp8,0,1.8245654106140137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,2,128,1,float16,float16,0,1.4857227007548015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,2,128,1,float16,fp8,0,1.4887253443400066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,2,128,1,fp8,fp8,0,1.4188532829284668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,4,128,1,float16,fp8,0,1.4969493548075359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,4,128,1,float16,float16,0,1.497221310933431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,4,128,1,fp8,fp8,0,1.4390880266825359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,8,128,1,float16,float16,0,1.5215999285380046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,8,128,1,float16,fp8,0,1.5322292645772297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,64,8,128,1,fp8,fp8,0,1.4690292676289876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,1,128,1,float16,float16,0,0.7519466876983643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,64,128,1,float16,float16,0,1.0509493350982666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,1,128,1,float16,fp8,0,0.7536426385243734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,1,128,1,fp8,fp8,0,0.695525328318278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,2,128,1,float16,float16,0,0.7560746669769287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,2,128,1,float16,fp8,0,0.7562133471171061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,64,128,1,float16,fp8,0,1.0045599937438965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,64,128,1,fp8,fp8,0,0.9201386769612631
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,2,128,1,fp8,fp8,0,0.7047893206278483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,4,128,1,float16,float16,0,0.7613066832224528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,4,128,1,float16,fp8,0,0.7606720129648844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,4,128,1,fp8,fp8,0,0.7276906967163086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,8,128,1,float16,fp8,0,0.7725919882456461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,1,128,1,float16,float16,0,0.38818132877349854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,8,128,1,float16,float16,0,0.7734080155690511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,64,8,128,1,fp8,fp8,0,0.7443946997324625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,64,128,1,float16,float16,0,0.5278773307800293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,64,128,1,float16,fp8,0,0.5069439808527628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,1,128,1,float16,fp8,0,0.3877333402633667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,64,128,1,fp8,fp8,0,0.4638880093892415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,1,128,1,fp8,fp8,0,0.3601866563161214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,2,128,1,float16,float16,0,0.3894079923629761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,2,128,1,float16,fp8,0,0.39133334159851074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,2,128,1,fp8,fp8,0,0.3615093231201172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,4,128,1,float16,fp8,0,0.39293332894643146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,4,128,1,float16,float16,0,0.3934239943822225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,4,128,1,fp8,fp8,0,0.3648639917373657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,8,128,1,float16,float16,0,0.3991626501083374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,8,128,1,fp8,fp8,0,0.3717706600824992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,64,8,128,1,float16,fp8,0,0.3986133337020874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,1,128,1,float16,fp8,0,0.20566932360331217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,1,128,1,float16,float16,0,0.20484799146652222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,64,128,1,float16,float16,0,0.277237335840861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,1,128,1,fp8,fp8,0,0.1922453244527181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,64,128,1,float16,fp8,0,0.268010675907135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,64,128,1,fp8,fp8,0,0.24036800861358643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,2,128,1,float16,fp8,0,0.2068586746851603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,2,128,1,float16,float16,0,0.2071946660677592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,2,128,1,fp8,fp8,0,0.19214399655659994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,4,128,1,float16,float16,0,0.2082293430964152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,4,128,1,float16,fp8,0,0.2095253268877665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,4,128,1,fp8,fp8,0,0.193338672320048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,8,128,1,float16,float16,0,0.21273066600163779
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,8,128,1,float16,fp8,0,0.2116533319155375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,64,8,128,1,fp8,fp8,0,0.19675199190775552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,64,128,1,float16,float16,0,0.15082666277885437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,64,128,1,float16,fp8,0,0.14576533436775208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,1,128,1,float16,float16,0,0.11346133550008138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,64,128,1,fp8,fp8,0,0.13245866696039835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,1,128,1,float16,fp8,0,0.11356266339619954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,1,128,1,fp8,fp8,0,0.10564800103505452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,2,128,1,float16,float16,0,0.11496000488599141
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,2,128,1,float16,fp8,0,0.11483200391133626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,2,128,1,fp8,fp8,0,0.10638399918874104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,4,128,1,float16,float16,0,0.11507733662923177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,4,128,1,float16,fp8,0,0.11613333225250244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,4,128,1,fp8,fp8,0,0.10746133327484131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,8,128,1,float16,float16,0,0.11739733815193176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,8,128,1,float16,fp8,0,0.11788266897201538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,64,8,128,1,fp8,fp8,0,0.10940266648928325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,64,128,1,float16,float16,0,0.08877333005269368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,1,128,1,float16,float16,0,0.06483200192451477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,1,128,1,float16,fp8,0,0.06538133323192596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,64,128,1,fp8,fp8,0,0.0759680022795995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,64,128,1,float16,fp8,0,0.08562666177749634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,1,128,1,fp8,fp8,0,0.05914666752020518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,2,128,1,float16,float16,0,0.06570133566856384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,2,128,1,float16,fp8,0,0.06471466521422069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,2,128,1,fp8,fp8,0,0.05905599892139435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,4,128,1,float16,float16,0,0.06577066580454509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,4,128,1,float16,fp8,0,0.06660800178845723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,4,128,1,fp8,fp8,0,0.06062933305899302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,8,128,1,float16,float16,0,0.06715733309586842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,8,128,1,float16,fp8,0,0.06709866722424825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,1,128,1,float16,fp8,0,0.039018665750821434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,64,8,128,1,fp8,fp8,0,0.06195199986298879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,1,128,1,float16,float16,0,0.039279999832312264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,64,128,1,float16,float16,0,0.04704533517360687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,64,128,1,float16,fp8,0,0.04593066871166229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,64,128,1,fp8,fp8,0,0.044693330923716225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,1,128,1,fp8,fp8,0,0.0360000009338061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,2,128,1,float16,float16,0,0.0390079990029335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,2,128,1,float16,fp8,0,0.03955200066169103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,2,128,1,fp8,fp8,0,0.03579200059175491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,4,128,1,fp8,fp8,0,0.037290667494138084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,4,128,1,float16,float16,0,0.04001066585381826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,4,128,1,float16,fp8,0,0.04043733328580856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,8,128,1,float16,float16,0,0.04004266609748205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,8,128,1,float16,fp8,0,0.040421334405740104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,64,8,128,1,fp8,fp8,0,0.03740799923737844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,1,128,1,float16,fp8,0,2.504271984100342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,1,128,1,float16,float16,0,2.5043412844340005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,1,128,1,fp8,fp8,0,2.461967945098877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,2,128,1,float16,float16,0,2.529733339945475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,2,128,1,float16,fp8,0,2.5376906394958496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,2,128,1,fp8,fp8,0,2.4636054039001465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,4,128,1,float16,float16,0,2.6430880228678384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,4,128,1,fp8,fp8,0,2.4992639223734536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,4,128,1,float16,fp8,0,2.648010730743408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,8,128,1,float16,float16,0,2.718501408894857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,8,128,1,float16,fp8,0,2.7102721532185874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,64,8,128,1,fp8,fp8,0,2.5706027348836265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,64,128,1,float16,float16,0,1.9390506744384766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,1,128,1,float16,float16,0,1.2637386322021484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,1,128,1,float16,fp8,0,1.264778693517049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,64,128,1,float16,fp8,0,1.847647984822591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,1,128,1,fp8,fp8,0,1.2281493345896404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,2,128,1,float16,float16,0,1.2700800100962322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,2,128,1,float16,fp8,0,1.2731253306070964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,2,128,1,fp8,fp8,0,1.2319786548614502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,4,128,1,float16,float16,0,1.2769920031229656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,4,128,1,float16,fp8,0,1.279642661412557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,4,128,1,fp8,fp8,0,1.2505119641621907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,64,128,1,fp8,fp8,0,1.6524052619934082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,8,128,1,float16,float16,0,1.3028266429901123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,1,128,1,float16,float16,0,0.6438773473103842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,1,128,1,float16,fp8,0,0.6444480021794637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,8,128,1,float16,fp8,0,1.3028106689453125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,64,8,128,1,fp8,fp8,0,1.2894720236460369
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,1,128,1,fp8,fp8,0,0.600383996963501
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,64,128,1,float16,float16,0,0.9720426400502523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,2,128,1,float16,float16,0,0.6484426657358805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,64,128,1,fp8,fp8,0,0.83133864402771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,64,128,1,float16,fp8,0,0.9204320112864176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,2,128,1,float16,fp8,0,0.6471786499023438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,2,128,1,fp8,fp8,0,0.6156800190607706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,4,128,1,float16,float16,0,0.6512213150660197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,4,128,1,float16,fp8,0,0.6513813336690267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,4,128,1,fp8,fp8,0,0.6347200075785319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,8,128,1,float16,float16,0,0.6654133399327596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,8,128,1,float16,fp8,0,0.6625279982884725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,64,8,128,1,fp8,fp8,0,0.6495413382848104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,1,128,1,float16,float16,0,0.3323093255360921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,1,128,1,float16,fp8,0,0.33187733093897503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,1,128,1,fp8,fp8,0,0.3128319978713989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,64,128,1,float16,float16,0,0.4886560042699178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,2,128,1,float16,float16,0,0.33326399326324463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,64,128,1,float16,fp8,0,0.4649440050125122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,2,128,1,float16,fp8,0,0.3327946662902832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,64,128,1,fp8,fp8,0,0.42183466752370197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,2,128,1,fp8,fp8,0,0.3155413269996643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,4,128,1,float16,fp8,0,0.33614933490753174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,4,128,1,float16,float16,0,0.3359573284784953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,4,128,1,fp8,fp8,0,0.3184906641642253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,8,128,1,float16,float16,0,0.34357333183288574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,8,128,1,fp8,fp8,0,0.3259306748708089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,64,8,128,1,float16,fp8,0,0.3415040175120036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,64,128,1,float16,float16,0,0.25728533665339154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,1,128,1,float16,float16,0,0.17589867115020752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,1,128,1,float16,fp8,0,0.17670933405558267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,1,128,1,fp8,fp8,0,0.16807466745376587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,2,128,1,float16,float16,0,0.17683200041453043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,64,128,1,fp8,fp8,0,0.2206826607386271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,64,128,1,float16,fp8,0,0.2461599906285604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,2,128,1,fp8,fp8,0,0.16931732495625815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,2,128,1,float16,fp8,0,0.1776319940884908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,4,128,1,float16,float16,0,0.17833600441614786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,4,128,1,float16,fp8,0,0.17864533265431723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,4,128,1,fp8,fp8,0,0.17115734020868936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,8,128,1,float16,float16,0,0.18253866831461588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,8,128,1,float16,fp8,0,0.18205867211023966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,64,8,128,1,fp8,fp8,0,0.17466666301091513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,64,128,1,float16,float16,0,0.14061866203943887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,1,128,1,float16,float16,0,0.09803733229637146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,64,128,1,float16,fp8,0,0.13532800475756326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,1,128,1,float16,fp8,0,0.09822400410970052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,64,128,1,fp8,fp8,0,0.12315199772516887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,1,128,1,fp8,fp8,0,0.09613333145777385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,2,128,1,float16,float16,0,0.09894399841626485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,2,128,1,float16,fp8,0,0.09941866993904114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,2,128,1,fp8,fp8,0,0.09554133812586467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,4,128,1,float16,float16,0,0.09934932986895244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,4,128,1,float16,fp8,0,0.09969600041707356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,4,128,1,fp8,fp8,0,0.09682666261990865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,8,128,1,float16,float16,0,0.10194666186968486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,8,128,1,float16,fp8,0,0.10207999746004741
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,64,8,128,1,fp8,fp8,0,0.09853333234786987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,64,128,1,float16,float16,0,0.087226668993632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,1,128,1,float16,float16,0,0.05862933397293091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,64,128,1,float16,fp8,0,0.08336533109347026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,1,128,1,float16,fp8,0,0.05861333509286245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,64,128,1,fp8,fp8,0,0.07255466779073079
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,1,128,1,fp8,fp8,0,0.05421333511670431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,2,128,1,float16,float16,0,0.05876266459623972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,2,128,1,float16,fp8,0,0.05870933334032694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,2,128,1,fp8,fp8,0,0.05443733433882395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,4,128,1,float16,float16,0,0.058415999015172325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,4,128,1,fp8,fp8,0,0.05507733424504598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,4,128,1,float16,fp8,0,0.05936000247796377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,8,128,1,float16,float16,0,0.060175999999046326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,8,128,1,float16,fp8,0,0.06002666552861532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,64,8,128,1,fp8,fp8,0,0.05685333410898844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,1,128,1,float16,fp8,0,0.036101333796978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,1,128,1,float16,float16,0,0.03669333209594091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,64,128,1,float16,float16,0,0.043738668163617454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,64,128,1,fp8,fp8,0,0.041008000572522484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,64,128,1,float16,fp8,0,0.04266666869322459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,1,128,1,fp8,fp8,0,0.03286933402220408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,2,128,1,float16,fp8,0,0.03583466758330663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,2,128,1,float16,float16,0,0.03579733272393545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,2,128,1,fp8,fp8,0,0.0332640012105306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,4,128,1,float16,fp8,0,0.0366293340921402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,4,128,1,float16,float16,0,0.03675200045108795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,4,128,1,fp8,fp8,0,0.03461866577466329
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,8,128,1,float16,float16,0,0.037658666570981346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,8,128,1,fp8,fp8,0,0.034629332522551216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,64,8,128,1,float16,fp8,0,0.03721600025892258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,1,128,1,float16,float16,0,0.02554133286078771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,64,128,1,float16,fp8,0,0.028197333216667175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,64,128,1,float16,float16,0,0.02812266598145167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,1,128,1,float16,fp8,0,0.025600001215934753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,64,128,1,fp8,fp8,0,0.025087999800841015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,1,128,1,fp8,fp8,0,0.023418667415777843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,2,128,1,float16,float16,0,0.025781333446502686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,2,128,1,fp8,fp8,0,0.02334933231274287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,4,128,1,float16,float16,0,0.025626666843891144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,2,128,1,float16,fp8,0,0.025936000049114227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,4,128,1,float16,fp8,0,0.02624533325433731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,8,128,1,float16,float16,0,0.02613866577545802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,4,128,1,fp8,fp8,0,0.023962666591008503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,8,128,1,float16,fp8,0,0.02643733223279317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,64,8,128,1,fp8,fp8,0,0.024400000770886738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,1,128,1,float16,fp8,0,1.1570773124694824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,1,128,1,float16,float16,0,1.1547839641571045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,1,128,1,fp8,fp8,0,1.218773365020752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,2,128,1,float16,float16,0,1.1625653107961018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,2,128,1,float16,fp8,0,1.1633013089497883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,2,128,1,fp8,fp8,0,1.22380264600118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,4,128,1,float16,float16,0,1.169648011525472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,4,128,1,float16,fp8,0,1.1702773571014404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,4,128,1,fp8,fp8,0,1.2578667004903157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,8,128,1,float16,float16,0,1.1959893703460693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,8,128,1,float16,fp8,0,1.1933333079020183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,64,128,1,float16,float16,0,0.9965333143870035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,1,128,1,float16,float16,0,0.5923786560694376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,1,128,1,float16,fp8,0,0.5928266843159994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,64,128,1,float16,fp8,0,0.9468533198038737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,64,8,128,1,fp8,fp8,0,1.2801547050476074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,1,128,1,fp8,fp8,0,0.6013333400090536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,2,128,1,float16,float16,0,0.5947680075963339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,2,128,1,float16,fp8,0,0.5944000085194906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,2,128,1,fp8,fp8,0,0.6080213387807211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,4,128,1,float16,float16,0,0.5975199937820435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,64,128,1,fp8,fp8,0,0.8360746701558431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,4,128,1,float16,fp8,0,0.5973440011342367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,4,128,1,fp8,fp8,0,0.6315199931462606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,1,128,1,float16,float16,0,0.30798399448394775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,8,128,1,fp8,fp8,0,0.6425546805063883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,8,128,1,float16,fp8,0,0.609392007191976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,64,8,128,1,float16,float16,0,0.611573338508606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,64,128,1,float16,float16,0,0.5056480169296265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,64,128,1,float16,fp8,0,0.48396265506744385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,1,128,1,float16,fp8,0,0.30826665957768756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,64,128,1,fp8,fp8,0,0.4145866632461548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,1,128,1,fp8,fp8,0,0.3123520016670227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,2,128,1,float16,float16,0,0.30908799171447754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,2,128,1,float16,fp8,0,0.3089226682980855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,2,128,1,fp8,fp8,0,0.3152800003687541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,4,128,1,float16,float16,0,0.31100799640019733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,4,128,1,float16,fp8,0,0.3113866647084554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,4,128,1,fp8,fp8,0,0.31780799229939777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,8,128,1,float16,float16,0,0.3192800084749858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,8,128,1,fp8,fp8,0,0.326581339041392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,64,8,128,1,float16,fp8,0,0.31837334235509235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,1,128,1,float16,float16,0,0.16510933637619019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,1,128,1,float16,fp8,0,0.16556800405184427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,64,128,1,float16,float16,0,0.2661760052045186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,1,128,1,fp8,fp8,0,0.16750399271647134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,2,128,1,float16,float16,0,0.16565866271654764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,64,128,1,float16,fp8,0,0.25365867217381793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,64,128,1,fp8,fp8,0,0.2207733392715454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,2,128,1,float16,fp8,0,0.16561599572499594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,2,128,1,fp8,fp8,0,0.16921599706013998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,4,128,1,float16,float16,0,0.16767466068267822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,4,128,1,float16,fp8,0,0.16803733507792154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,4,128,1,fp8,fp8,0,0.17081065972646078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,8,128,1,float16,float16,0,0.17123732964197794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,8,128,1,float16,fp8,0,0.17164266109466553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,64,8,128,1,fp8,fp8,0,0.17375999689102173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,1,128,1,float16,float16,0,0.09418132901191711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,64,128,1,float16,float16,0,0.14604266484578451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,1,128,1,float16,fp8,0,0.0940053363641103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,64,128,1,float16,fp8,0,0.13820266723632812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,64,128,1,fp8,fp8,0,0.119759996732076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,1,128,1,fp8,fp8,0,0.09526933232943217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,2,128,1,float16,float16,0,0.09353599945704143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,2,128,1,float16,fp8,0,0.0946560005346934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,2,128,1,fp8,fp8,0,0.09556266665458679
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,4,128,1,float16,float16,0,0.09449066718419392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,4,128,1,float16,fp8,0,0.09477333227793376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,4,128,1,fp8,fp8,0,0.09610666831334432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,8,128,1,float16,fp8,0,0.09699199597040813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,8,128,1,float16,float16,0,0.09641599655151367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,64,8,128,1,fp8,fp8,0,0.0985599954922994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,64,128,1,float16,float16,0,0.08333333333333333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,64,128,1,float16,fp8,0,0.08020799855391185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,1,128,1,float16,float16,0,0.055567999680837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,64,128,1,fp8,fp8,0,0.07163199782371521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,1,128,1,float16,fp8,0,0.055488000313440956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,2,128,1,float16,float16,0,0.05588266750176748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,2,128,1,float16,fp8,0,0.05571199953556061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,2,128,1,fp8,fp8,0,0.05390933156013489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,4,128,1,float16,float16,0,0.05611733098824819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,1,128,1,fp8,fp8,0,0.05459733307361603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,4,128,1,float16,fp8,0,0.055904000997543335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,4,128,1,fp8,fp8,0,0.05532266696294149
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,8,128,1,float16,float16,0,0.05736533304055532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,8,128,1,float16,fp8,0,0.05751466751098633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,64,8,128,1,fp8,fp8,0,0.05573866764704386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,64,128,1,float16,float16,0,0.04192533095677694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,64,128,1,fp8,fp8,0,0.041120000183582306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,1,128,1,float16,float16,0,0.03448000053564707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,1,128,1,float16,fp8,0,0.03443733354409536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,64,128,1,float16,fp8,0,0.03997866561015447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,1,128,1,fp8,fp8,0,0.03278933217128118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,2,128,1,float16,float16,0,0.034058667719364166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,2,128,1,float16,fp8,0,0.03412266572316488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,2,128,1,fp8,fp8,0,0.03278933217128118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,4,128,1,float16,float16,0,0.03496533383925756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,4,128,1,float16,fp8,0,0.035162667433420815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,8,128,1,float16,float16,0,0.03612266729275385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,8,128,1,float16,fp8,0,0.03545066714286804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,8,128,1,fp8,fp8,0,0.0344106654326121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,64,4,128,1,fp8,fp8,0,0.034815999368826546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,64,128,1,float16,float16,0,0.025989333788553875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,64,128,1,float16,fp8,0,0.027210667729377747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,1,128,1,float16,float16,0,0.0245919997493426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,64,128,1,fp8,fp8,0,0.025498665869235992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,1,128,1,float16,fp8,0,0.024693332612514496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,1,128,1,fp8,fp8,0,0.022986667851607006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,2,128,1,float16,fp8,0,0.024298667907714844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,2,128,1,fp8,fp8,0,0.02293866624434789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,4,128,1,float16,float16,0,0.02515200028816859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,4,128,1,fp8,fp8,0,0.023775999744733173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,4,128,1,float16,fp8,0,0.024714666108290356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,2,128,1,float16,float16,0,0.024570666253566742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,8,128,1,float16,float16,0,0.024826665719350178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,8,128,1,float16,fp8,0,0.02513066679239273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,64,8,128,1,fp8,fp8,0,0.023914667467276256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,64,128,1,float16,float16,0,0.021194666624069214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,64,128,1,fp8,fp8,0,0.0215786670645078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,1,128,1,float16,fp8,0,0.02109866589307785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,2,128,1,float16,fp8,0,0.020741333564122517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,2,128,1,float16,float16,0,0.020608000457286835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,64,128,1,float16,fp8,0,0.02128533273935318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,1,128,1,fp8,fp8,0,0.020303999384244282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,1,128,1,float16,float16,0,0.020256000260512035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,2,128,1,fp8,fp8,0,0.020608000457286835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,4,128,1,float16,float16,0,0.02089066555102666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,4,128,1,float16,fp8,0,0.02080533280968666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,4,128,1,fp8,fp8,0,0.021040000021457672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,8,128,1,float16,fp8,0,0.021114667256673176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,8,128,1,float16,float16,0,0.02067733307679494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,64,8,128,1,fp8,fp8,0,0.02110933264096578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,1,128,1,float16,float16,0,0.7526773611704508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,1,128,1,fp8,fp8,0,0.7984373569488525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,1,128,1,float16,fp8,0,0.7535786628723145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,2,128,1,float16,fp8,0,0.7558186848958334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,2,128,1,float16,float16,0,0.7553813457489014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,4,128,1,float16,float16,0,0.7597013314565023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,2,128,1,fp8,fp8,0,0.8158453305562338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,4,128,1,float16,fp8,0,0.7599786917368571
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,4,128,1,fp8,fp8,0,0.8470933437347412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,8,128,1,float16,float16,0,0.7719146410624186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,8,128,1,fp8,fp8,0,0.8681973616282145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,64,8,128,1,float16,fp8,0,0.7683680057525635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,1,128,1,float16,float16,0,0.3840746482213338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,1,128,1,fp8,fp8,0,0.41375998655955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,1,128,1,float16,fp8,0,0.3845119873682658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,2,128,1,float16,float16,0,0.3855146567026774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,64,128,1,float16,float16,0,0.5076640049616495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,2,128,1,float16,fp8,0,0.38591468334198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,64,128,1,float16,fp8,0,0.48425066471099854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,64,128,1,fp8,fp8,0,0.5221066474914551
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,2,128,1,fp8,fp8,0,0.41860798994700116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,4,128,1,float16,fp8,0,0.3886186679204305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,4,128,1,float16,float16,0,0.387824018796285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,4,128,1,fp8,fp8,0,0.4261920054753621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,8,128,1,float16,float16,0,0.3948640028635661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,8,128,1,fp8,fp8,0,0.4346880118052165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,64,8,128,1,float16,fp8,0,0.39290134112040204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,1,128,1,float16,float16,0,0.20091732343037924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,64,128,1,float16,float16,0,0.2659839987754822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,64,128,1,fp8,fp8,0,0.26998400688171387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,1,128,1,float16,fp8,0,0.20120000839233398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,64,128,1,float16,fp8,0,0.2541653315226237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,1,128,1,fp8,fp8,0,0.21954667568206787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,2,128,1,float16,float16,0,0.20075732469558716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,2,128,1,fp8,fp8,0,0.22058133284250894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,4,128,1,float16,float16,0,0.2030506730079651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,2,128,1,float16,fp8,0,0.2015626629193624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,4,128,1,float16,fp8,0,0.20296533902486166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,4,128,1,fp8,fp8,0,0.22262932856877646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,8,128,1,float16,float16,0,0.207914670308431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,8,128,1,float16,fp8,0,0.20654400189717612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,1,128,1,float16,float16,0,0.109333336353302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,1,128,1,float16,fp8,0,0.1090773344039917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,64,8,128,1,fp8,fp8,0,0.22631466388702393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,1,128,1,fp8,fp8,0,0.12168000141779582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,64,128,1,float16,float16,0,0.14327466487884521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,64,128,1,fp8,fp8,0,0.1454026699066162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,64,128,1,float16,fp8,0,0.1381600002447764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,2,128,1,float16,float16,0,0.10891733566919963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,2,128,1,float16,fp8,0,0.10925333698590596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,2,128,1,fp8,fp8,0,0.12158399820327759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,4,128,1,float16,fp8,0,0.1102239986260732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,4,128,1,float16,float16,0,0.1095413366953532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,4,128,1,fp8,fp8,0,0.12125866611798604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,8,128,1,float16,float16,0,0.11203199625015259
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,8,128,1,fp8,fp8,0,0.12481600046157837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,64,8,128,1,float16,fp8,0,0.11152000228563945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,64,128,1,float16,float16,0,0.08216533561547597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,64,128,1,fp8,fp8,0,0.08283199866612752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,1,128,1,float16,float16,0,0.06185600161552429
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,64,128,1,float16,fp8,0,0.07958933214346568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,1,128,1,fp8,fp8,0,0.06841066479682922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,1,128,1,float16,fp8,0,0.06166933476924896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,2,128,1,float16,fp8,0,0.06244266529877981
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,2,128,1,float16,float16,0,0.0620959997177124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,2,128,1,fp8,fp8,0,0.06794666747252147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,4,128,1,float16,fp8,0,0.06282133360703786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,4,128,1,float16,float16,0,0.06246933341026306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,4,128,1,fp8,fp8,0,0.06938133140405019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,8,128,1,float16,fp8,0,0.06392000118891399
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,8,128,1,fp8,fp8,0,0.07029866675535838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,64,128,1,float16,fp8,0,0.04166933397452036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,64,128,1,fp8,fp8,0,0.0484746644894282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,64,8,128,1,float16,float16,0,0.06342400113741557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,64,128,1,float16,float16,0,0.04278400043646494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,1,128,1,float16,float16,0,0.03775466730197271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,1,128,1,float16,fp8,0,0.03774933268626531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,1,128,1,fp8,fp8,0,0.04027199993530909
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,2,128,1,float16,float16,0,0.037605332831541695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,2,128,1,float16,fp8,0,0.03804266701141993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,4,128,1,fp8,fp8,0,0.042122667034467064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,4,128,1,float16,float16,0,0.03835733234882355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,4,128,1,float16,fp8,0,0.03865066667397817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,8,128,1,float16,float16,0,0.03836799909671148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,2,128,1,fp8,fp8,0,0.04058666775623957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,8,128,1,float16,fp8,0,0.03857066730658213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,64,8,128,1,fp8,fp8,0,0.042538667718569435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,64,128,1,float16,float16,0,0.02805333336194356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,64,128,1,float16,fp8,0,0.0277813325325648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,64,128,1,fp8,fp8,0,0.029648000995318096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,1,128,1,float16,float16,0,0.025568000972270966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,1,128,1,fp8,fp8,0,0.026682667434215546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,2,128,1,float16,float16,0,0.026170666019121807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,2,128,1,float16,fp8,0,0.02622933437426885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,1,128,1,float16,fp8,0,0.026159999271233875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,2,128,1,fp8,fp8,0,0.026767998933792114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,4,128,1,float16,float16,0,0.026447998980681103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,4,128,1,float16,fp8,0,0.026074667771657307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,4,128,1,fp8,fp8,0,0.027850667635599773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,8,128,1,float16,float16,0,0.02644266684850057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,8,128,1,float16,fp8,0,0.026517334083716076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,64,128,1,float16,float16,0,0.02139200021823247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,1,128,1,float16,float16,0,0.020015999674797058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,64,128,1,float16,fp8,0,0.021066665649414062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,64,128,1,fp8,fp8,0,0.021685334543387096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,64,8,128,1,fp8,fp8,0,0.028042666614055634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,1,128,1,float16,fp8,0,0.020010666300853092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,1,128,1,fp8,fp8,0,0.020154666155576706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,2,128,1,float16,fp8,0,0.020554666717847187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,2,128,1,float16,float16,0,0.020202666521072388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,2,128,1,fp8,fp8,0,0.020506666352351505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,4,128,1,float16,float16,0,0.020128000527620316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,4,128,1,fp8,fp8,0,0.021087999145189922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,4,128,1,float16,fp8,0,0.0206986665725708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,8,128,1,float16,float16,0,0.020138667275508244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,8,128,1,float16,fp8,0,0.020682666450738907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,64,8,128,1,fp8,fp8,0,0.021365332106749218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,64,128,1,float16,float16,0,0.0186666672428449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,64,128,1,float16,fp8,0,0.018613333503405254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,64,128,1,fp8,fp8,0,0.019930666933457058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,1,128,1,float16,fp8,0,0.018522666146357853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,1,128,1,float16,float16,0,0.018405333161354065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,1,128,1,fp8,fp8,0,0.019519999623298645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,2,128,1,float16,fp8,0,0.018810667097568512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,2,128,1,float16,float16,0,0.01812800019979477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,2,128,1,fp8,fp8,0,0.019189332922299702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,4,128,1,float16,float16,0,0.01848000039656957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,4,128,1,float16,fp8,0,0.01874133323629697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,8,128,1,float16,float16,0,0.018661333868900936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,8,128,1,float16,fp8,0,0.01887999971707662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,8,128,1,fp8,fp8,0,0.019823999454577763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,64,4,128,1,fp8,fp8,0,0.01987733319401741
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,1,128,1,float16,float16,0,0.5121599833170573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,1,128,1,float16,fp8,0,0.5130293369293213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,1,128,1,fp8,fp8,0,0.6477440198262533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,2,128,1,float16,float16,0,0.5183200041453043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,2,128,1,float16,fp8,0,0.518122673034668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,2,128,1,fp8,fp8,0,0.6529813210169474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,4,128,1,float16,float16,0,0.5277599891026815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,4,128,1,fp8,fp8,0,0.6586026748021444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,4,128,1,float16,fp8,0,0.5282133420308431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,8,128,1,float16,float16,0,0.5425333182017008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,8,128,1,float16,fp8,0,0.5426506598790487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,64,8,128,1,fp8,fp8,0,0.6689706643422445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,64,128,1,float16,float16,0,0.33326399326324463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,1,128,1,float16,float16,0,0.2627466718355815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,64,128,1,float16,fp8,0,0.32210665941238403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,1,128,1,float16,fp8,0,0.2643893361091614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,1,128,1,fp8,fp8,0,0.3365333477656047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,2,128,1,float16,float16,0,0.266213337580363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,2,128,1,float16,fp8,0,0.26658133665720624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,64,128,1,fp8,fp8,0,0.3849173386891683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,2,128,1,fp8,fp8,0,0.3389600118001302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,4,128,1,float16,fp8,0,0.2720639904340108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,4,128,1,float16,float16,0,0.2715946634610494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,4,128,1,fp8,fp8,0,0.3422559897104899
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,8,128,1,float16,float16,0,0.2793546716372172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,8,128,1,float16,fp8,0,0.27911466360092163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,64,8,128,1,fp8,fp8,0,0.34508268038431805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,1,128,1,float16,float16,0,0.14039466778437296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,1,128,1,float16,fp8,0,0.1404213309288025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,64,128,1,float16,fp8,0,0.16777600844701132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,64,128,1,float16,float16,0,0.17458132902781168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,2,128,1,float16,float16,0,0.14086932937304178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,64,128,1,fp8,fp8,0,0.20494933923085532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,1,128,1,fp8,fp8,0,0.18073066075642905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,2,128,1,float16,fp8,0,0.141375998655955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,2,128,1,fp8,fp8,0,0.18171733617782593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,4,128,1,float16,float16,0,0.14355199535687765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,4,128,1,float16,fp8,0,0.14334932963053384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,4,128,1,fp8,fp8,0,0.18102399508158365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,8,128,1,fp8,fp8,0,0.18401066462198892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,8,128,1,float16,float16,0,0.14847999811172485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,64,128,1,float16,float16,0,0.09778133034706116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,64,128,1,float16,fp8,0,0.0946720043818156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,64,8,128,1,float16,fp8,0,0.14732799927393594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,64,128,1,fp8,fp8,0,0.11294399698575337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,1,128,1,float16,fp8,0,0.07753066718578339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,1,128,1,float16,float16,0,0.0771679977575938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,2,128,1,float16,float16,0,0.07753600180149078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,2,128,1,float16,fp8,0,0.07763733466466267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,2,128,1,fp8,fp8,0,0.09983999530474345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,4,128,1,float16,float16,0,0.07755733529726665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,1,128,1,fp8,fp8,0,0.10002133250236511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,4,128,1,float16,fp8,0,0.07789866626262665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,4,128,1,fp8,fp8,0,0.1002346674601237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,8,128,1,float16,float16,0,0.07863999903202057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,8,128,1,float16,fp8,0,0.07881600161393483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,64,8,128,1,fp8,fp8,0,0.10094933708508809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,64,128,1,float16,float16,0,0.05065600077311198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,64,128,1,float16,fp8,0,0.04877333343029022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,1,128,1,float16,float16,0,0.04567466676235199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,1,128,1,fp8,fp8,0,0.05589866638183594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,1,128,1,float16,fp8,0,0.04677866895993551
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,64,128,1,fp8,fp8,0,0.06464000046253204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,2,128,1,float16,fp8,0,0.045935998360315956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,4,128,1,float16,float16,0,0.0467199981212616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,4,128,1,float16,fp8,0,0.04764266808827718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,2,128,1,float16,float16,0,0.046480000019073486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,4,128,1,fp8,fp8,0,0.05900266766548157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,2,128,1,fp8,fp8,0,0.05653866628805796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,8,128,1,float16,float16,0,0.04650133351484934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,8,128,1,float16,fp8,0,0.047354668378829956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,64,128,1,float16,float16,0,0.031119999786218006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,64,8,128,1,fp8,fp8,0,0.05909866591294607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,64,128,1,fp8,fp8,0,0.03737066686153412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,1,128,1,float16,fp8,0,0.029663999875386555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,1,128,1,fp8,fp8,0,0.03541333228349686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,1,128,1,float16,float16,0,0.030095999439557392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,2,128,1,float16,float16,0,0.02993600070476532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,64,128,1,float16,fp8,0,0.030986666679382324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,2,128,1,float16,fp8,0,0.030063999195893604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,2,128,1,fp8,fp8,0,0.03568000098069509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,4,128,1,float16,float16,0,0.030074665943781536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,4,128,1,float16,fp8,0,0.030159999926884968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,8,128,1,float16,float16,0,0.030042665700117748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,8,128,1,fp8,fp8,0,0.03666666646798452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,4,128,1,fp8,fp8,0,0.03618666778008143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,64,8,128,1,float16,fp8,0,0.030943999687830608
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,64,128,1,float16,float16,0,0.0220320001244545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,64,128,1,float16,fp8,0,0.022437334060668945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,1,128,1,float16,float16,0,0.02120000123977661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,64,128,1,fp8,fp8,0,0.025370667378107708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,1,128,1,float16,fp8,0,0.02181866765022278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,2,128,1,float16,float16,0,0.021573332448800404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,2,128,1,float16,fp8,0,0.021989333132902782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,1,128,1,fp8,fp8,0,0.02409599969784419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,2,128,1,fp8,fp8,0,0.02492266645034154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,4,128,1,float16,float16,0,0.02160000056028366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,4,128,1,fp8,fp8,0,0.025087999800841015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,4,128,1,float16,fp8,0,0.021829334398110706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,8,128,1,float16,fp8,0,0.02203733225663503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,8,128,1,float16,float16,0,0.021712000171343487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,64,128,1,float16,float16,0,0.01823466643691063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,64,128,1,float16,fp8,0,0.018687999496857326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,64,8,128,1,fp8,fp8,0,0.02537599951028824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,64,128,1,fp8,fp8,0,0.020010666300853092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,1,128,1,float16,fp8,0,0.01821333294113477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,1,128,1,float16,float16,0,0.017375999440749485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,1,128,1,fp8,fp8,0,0.01930133377512296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,2,128,1,float16,fp8,0,0.01834133391578992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,2,128,1,float16,float16,0,0.01786133274435997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,4,128,1,float16,float16,0,0.018063999712467194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,4,128,1,float16,fp8,0,0.018309333672126133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,2,128,1,fp8,fp8,0,0.01952533299724261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,4,128,1,fp8,fp8,0,0.019882666567961376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,8,128,1,float16,float16,0,0.017866666118303936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,8,128,1,float16,fp8,0,0.01829333355029424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,64,8,128,1,fp8,fp8,0,0.020213333268960316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,64,128,1,float16,float16,0,0.017093333105246227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,64,128,1,float16,fp8,0,0.017743999759356182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,1,128,1,float16,float16,0,0.01710933322707812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,1,128,1,float16,fp8,0,0.017450666675964992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,1,128,1,fp8,fp8,0,0.01882133384545644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,64,128,1,fp8,fp8,0,0.01899733394384384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,2,128,1,float16,fp8,0,0.017557332913080852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,2,128,1,float16,float16,0,0.01743999992807706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,2,128,1,fp8,fp8,0,0.019621333728233974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,4,128,1,float16,float16,0,0.016997333616018295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,4,128,1,float16,fp8,0,0.01747200017174085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,4,128,1,fp8,fp8,0,0.01916266605257988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,8,128,1,float16,fp8,0,0.01788266624013583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,8,128,1,fp8,fp8,0,0.018719999740521114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,64,8,128,1,float16,float16,0,0.01709866647919019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,1,128,1,float16,fp8,0,0.43462932109832764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,1,128,1,float16,float16,0,0.43426668643951416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,1,128,1,fp8,fp8,0,0.5730666716893514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,2,128,1,float16,float16,0,0.4374613364537557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,2,128,1,float16,fp8,0,0.43749332427978516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,2,128,1,fp8,fp8,0,0.5752586523691813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,4,128,1,float16,float16,0,0.4419999917348226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,4,128,1,float16,fp8,0,0.4426240126291911
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,4,128,1,fp8,fp8,0,0.5827786525090536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,8,128,1,float16,float16,0,0.44950934251149494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,8,128,1,float16,fp8,0,0.45004268487294513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,64,8,128,1,fp8,fp8,0,0.5861813227335612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,64,128,1,float16,float16,0,0.26844799518585205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,64,128,1,float16,fp8,0,0.26411734024683636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,1,128,1,float16,float16,0,0.22566932439804077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,1,128,1,float16,fp8,0,0.2259999910990397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,1,128,1,fp8,fp8,0,0.30034132798512775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,2,128,1,float16,float16,0,0.2263466715812683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,64,128,1,fp8,fp8,0,0.32102400064468384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,2,128,1,float16,fp8,0,0.22728532552719116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,4,128,1,float16,float16,0,0.2289173404375712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,2,128,1,fp8,fp8,0,0.30165332555770874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,4,128,1,fp8,fp8,0,0.3011786739031474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,4,128,1,float16,fp8,0,0.22829866409301758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,8,128,1,float16,float16,0,0.23186665773391724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,8,128,1,fp8,fp8,0,0.3038453261057536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,64,8,128,1,float16,fp8,0,0.23173334201176962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,64,128,1,float16,float16,0,0.13140799601872763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,1,128,1,float16,float16,0,0.12029332915941875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,1,128,1,float16,fp8,0,0.12109333276748657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,1,128,1,fp8,fp8,0,0.16055466731389365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,64,128,1,fp8,fp8,0,0.17230399449666342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,2,128,1,float16,float16,0,0.12070932984352112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,64,128,1,float16,fp8,0,0.1288800040880839
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,2,128,1,float16,fp8,0,0.12076266606648763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,2,128,1,fp8,fp8,0,0.159360001484553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,4,128,1,float16,float16,0,0.12079999844233195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,4,128,1,float16,fp8,0,0.12168533603350322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,8,128,1,float16,float16,0,0.12171199917793274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,4,128,1,fp8,fp8,0,0.1612160007158915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,8,128,1,float16,fp8,0,0.12152533729871114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,64,8,128,1,fp8,fp8,0,0.16351466377576193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,1,128,1,float16,float16,0,0.06653866668542226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,64,128,1,float16,float16,0,0.06881066660086314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,1,128,1,float16,fp8,0,0.06694933275381725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,64,128,1,fp8,fp8,0,0.09569600224494934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,64,128,1,float16,fp8,0,0.06755733489990234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,1,128,1,fp8,fp8,0,0.08866666754086812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,2,128,1,float16,float16,0,0.06724266707897186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,2,128,1,float16,fp8,0,0.06712000072002411
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,2,128,1,fp8,fp8,0,0.08879466851552327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,4,128,1,float16,float16,0,0.06758399804433186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,4,128,1,float16,fp8,0,0.06806933383146922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,4,128,1,fp8,fp8,0,0.09009599685668945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,8,128,1,fp8,fp8,0,0.09016533692677815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,8,128,1,float16,fp8,0,0.06822933256626129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,64,128,1,float16,float16,0,0.040021332601706185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,64,128,1,float16,fp8,0,0.040549332896868386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,64,8,128,1,float16,float16,0,0.0677706648906072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,64,128,1,fp8,fp8,0,0.05236266553401947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,1,128,1,float16,float16,0,0.04223466912905375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,1,128,1,float16,fp8,0,0.041797334949175514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,1,128,1,fp8,fp8,0,0.05197866757710775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,2,128,1,float16,float16,0,0.04171733558177948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,2,128,1,float16,fp8,0,0.04197333256403605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,4,128,1,float16,float16,0,0.04139200101296107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,2,128,1,fp8,fp8,0,0.05189333359400431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,4,128,1,fp8,fp8,0,0.05223466455936432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,8,128,1,float16,fp8,0,0.04196799794832865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,8,128,1,float16,float16,0,0.042490666111310325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,4,128,1,float16,fp8,0,0.042538667718569435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,64,128,1,float16,float16,0,0.02752000093460083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,64,8,128,1,fp8,fp8,0,0.05190399785836538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,64,128,1,float16,fp8,0,0.027471999327341717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,64,128,1,fp8,fp8,0,0.03355200091997782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,1,128,1,float16,fp8,0,0.02738133321205775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,2,128,1,float16,float16,0,0.02775999903678894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,1,128,1,float16,float16,0,0.027834666272004444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,2,128,1,float16,fp8,0,0.02758399893840154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,1,128,1,fp8,fp8,0,0.0329120010137558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,4,128,1,float16,float16,0,0.027488000690937042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,4,128,1,fp8,fp8,0,0.033600000043710075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,4,128,1,float16,fp8,0,0.02802666773398717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,2,128,1,fp8,fp8,0,0.03362133353948593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,8,128,1,float16,fp8,0,0.02805333336194356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,8,128,1,float16,float16,0,0.028079998989899952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,64,8,128,1,fp8,fp8,0,0.03334933271010717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,64,128,1,float16,float16,0,0.020319999506076176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,64,128,1,float16,fp8,0,0.02107733239730199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,64,128,1,fp8,fp8,0,0.0242399995525678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,1,128,1,float16,fp8,0,0.020527999848127365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,1,128,1,fp8,fp8,0,0.023520000278949738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,1,128,1,float16,float16,0,0.02089066555102666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,2,128,1,float16,fp8,0,0.021040000021457672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,2,128,1,float16,float16,0,0.020784000555674236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,2,128,1,fp8,fp8,0,0.023775999744733173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,4,128,1,float16,float16,0,0.021013334393501282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,4,128,1,float16,fp8,0,0.021189334491888683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,8,128,1,float16,fp8,0,0.02126399924357732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,64,128,1,float16,float16,0,0.017701332767804463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,8,128,1,fp8,fp8,0,0.023845332364241283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,8,128,1,float16,float16,0,0.021136000752449036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,64,4,128,1,fp8,fp8,0,0.0235359991590182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,64,128,1,fp8,fp8,0,0.019440000255902607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,64,128,1,float16,fp8,0,0.01811733345190684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,1,128,1,float16,float16,0,0.01757866640885671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,2,128,1,fp8,fp8,0,0.019098666807015736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,1,128,1,float16,fp8,0,0.018320000420014065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,1,128,1,fp8,fp8,0,0.019082666685183842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,4,128,1,float16,float16,0,0.01783466711640358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,2,128,1,float16,float16,0,0.0173333336909612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,4,128,1,fp8,fp8,0,0.019541333119074505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,4,128,1,float16,fp8,0,0.017664000391960144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,2,128,1,float16,fp8,0,0.0176959993938605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,8,128,1,float16,float16,0,0.017632000148296356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,8,128,1,fp8,fp8,0,0.019845332950353622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,64,8,128,1,float16,fp8,0,0.018186666071414948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,64,128,1,float16,fp8,0,0.017093333105246227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,64,128,1,fp8,fp8,0,0.01903466631968816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,1,128,1,float16,fp8,0,0.01718933383623759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,64,128,1,float16,float16,0,0.016506666938463848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,2,128,1,float16,fp8,0,0.017082666357358296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,1,128,1,fp8,fp8,0,0.01893866683046023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,1,128,1,float16,float16,0,0.01670933390657107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,2,128,1,float16,float16,0,0.016976000120242436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,2,128,1,fp8,fp8,0,0.018559999763965607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,4,128,1,float16,float16,0,0.016949333250522614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,4,128,1,fp8,fp8,0,0.01848000039656957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,8,128,1,float16,float16,0,0.016832000265518825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,4,128,1,float16,fp8,0,0.017317333569129307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,8,128,1,fp8,fp8,0,0.018751999984184902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,64,8,128,1,float16,fp8,0,0.01718933383623759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,1,128,1,fp8,fp8,0,33.07904561360677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,1,128,1,float16,float16,0,49.888936360677086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,1,128,1,float16,fp8,0,49.82037353515625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,2,128,1,fp8,fp8,0,32.80627695719401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,2,128,1,float16,float16,0,50.282511393229164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,2,128,1,float16,fp8,0,50.83919779459635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,4,128,1,float16,float16,0,51.95055135091146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,4,128,1,float16,fp8,0,50.35820515950521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,4,128,1,fp8,fp8,0,32.80279032389323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,8,128,1,fp8,fp8,0,33.02987162272135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,8,128,1,float16,fp8,0,50.16629536946615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,48,8,128,1,float16,float16,0,49.7633310953776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,48,128,1,float16,float16,0,25.202245076497395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,1,128,1,float16,float16,0,25.180404663085938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,1,128,1,fp8,fp8,0,16.492074330647785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,1,128,1,float16,fp8,0,25.164655049641926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,48,128,1,float16,fp8,0,25.585797627766926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,2,128,1,float16,float16,0,25.027631123860676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,2,128,1,float16,fp8,0,25.28570048014323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,2,128,1,fp8,fp8,0,16.425215403238933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,4,128,1,float16,float16,0,25.28912099202474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,4,128,1,float16,fp8,0,25.15728505452474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,48,128,1,fp8,fp8,0,16.984933217366535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,4,128,1,fp8,fp8,0,17.013968149820965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,8,128,1,float16,float16,0,25.064112345377605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,1,128,1,float16,float16,0,13.051429748535156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,8,128,1,float16,fp8,0,25.269541422526043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,1,128,1,float16,fp8,0,12.672288258870443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,48,8,128,1,fp8,fp8,0,16.497759501139324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,1,128,1,fp8,fp8,0,8.617274602254232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,48,128,1,float16,float16,0,12.649178822835287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,48,128,1,float16,fp8,0,12.667930603027344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,48,128,1,fp8,fp8,0,8.491615931193033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,2,128,1,float16,float16,0,12.877333323160807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,2,128,1,float16,fp8,0,12.562437693277994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,2,128,1,fp8,fp8,0,8.275199890136719
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,4,128,1,float16,float16,0,13.080027262369791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,4,128,1,float16,fp8,0,12.680330912272135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,4,128,1,fp8,fp8,0,8.458917617797852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,8,128,1,float16,float16,0,12.893423716227213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,1,128,1,float16,float16,0,6.362725575764974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,8,128,1,float16,fp8,0,12.69533920288086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,48,8,128,1,fp8,fp8,0,8.38095474243164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,48,128,1,float16,float16,0,6.126122792561849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,1,128,1,float16,fp8,0,6.2484588623046875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,48,128,1,fp8,fp8,0,4.28278923034668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,48,128,1,float16,fp8,0,6.536501566569011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,2,128,1,float16,float16,0,6.471935907999675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,2,128,1,fp8,fp8,0,4.2684478759765625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,1,128,1,fp8,fp8,0,4.286261240641276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,2,128,1,float16,fp8,0,6.403151830037435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,4,128,1,float16,float16,0,6.521930694580078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,4,128,1,float16,fp8,0,6.327487945556641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,4,128,1,fp8,fp8,0,4.26694393157959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,8,128,1,float16,float16,0,6.422741572062175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,8,128,1,float16,fp8,0,6.395749409993489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,48,8,128,1,fp8,fp8,0,4.262474695841472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,1,128,1,float16,float16,0,28.78522237141927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,1,128,1,float16,fp8,0,29.203099568684895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,1,128,1,fp8,fp8,0,19.357412974039715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,2,128,1,float16,float16,0,29.778416951497395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,2,128,1,fp8,fp8,0,19.352699279785156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,2,128,1,float16,fp8,0,29.199803670247395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,4,128,1,float16,float16,0,29.128196716308594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,4,128,1,fp8,fp8,0,19.61632537841797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,4,128,1,float16,fp8,0,28.636138916015625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,8,128,1,float16,float16,0,29.028335571289062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,8,128,1,fp8,fp8,0,19.39687983194987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,48,8,128,1,float16,fp8,0,29.58379618326823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,48,128,1,float16,float16,0,14.713823954264322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,1,128,1,float16,float16,0,14.465909322102865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,48,128,1,float16,fp8,0,14.706586201985678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,1,128,1,float16,fp8,0,14.473072052001953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,1,128,1,fp8,fp8,0,9.587205251057943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,2,128,1,float16,float16,0,14.952176411946615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,2,128,1,float16,fp8,0,14.600650787353516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,4,128,1,float16,float16,0,14.698912302652994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,2,128,1,fp8,fp8,0,9.579343795776367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,48,128,1,fp8,fp8,0,9.944517135620117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,4,128,1,float16,fp8,0,14.445679982503256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,4,128,1,fp8,fp8,0,9.834431966145834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,8,128,1,float16,float16,0,14.962292989095053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,1,128,1,float16,float16,0,7.399050394694011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,1,128,1,float16,fp8,0,7.402912139892578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,8,128,1,float16,fp8,0,14.609391530354818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,48,8,128,1,fp8,fp8,0,9.733882904052734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,1,128,1,fp8,fp8,0,4.815189361572266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,48,128,1,float16,fp8,0,7.306933085123698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,48,128,1,fp8,fp8,0,4.927498817443848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,48,128,1,float16,float16,0,7.275237401326497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,2,128,1,float16,fp8,0,7.2850189208984375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,2,128,1,float16,float16,0,7.551647822062175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,2,128,1,fp8,fp8,0,4.827322642008464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,4,128,1,float16,float16,0,7.371386845906575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,4,128,1,fp8,fp8,0,4.877546628316243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,4,128,1,float16,fp8,0,7.518149058024089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,8,128,1,float16,float16,0,7.380709330240886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,8,128,1,float16,fp8,0,7.318922678629558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,48,8,128,1,fp8,fp8,0,5.107568105061849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,1,128,1,float16,float16,0,3.6354452768961587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,1,128,1,float16,fp8,0,3.618266741434733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,48,128,1,float16,float16,0,3.642416000366211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,48,128,1,float16,fp8,0,3.6258986790974936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,48,128,1,fp8,fp8,0,2.611248016357422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,1,128,1,fp8,fp8,0,2.5753119786580405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,2,128,1,float16,float16,0,3.797365188598633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,2,128,1,float16,fp8,0,3.599818547566732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,2,128,1,fp8,fp8,0,2.5748373667399087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,4,128,1,float16,float16,0,3.7463626861572266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,4,128,1,float16,fp8,0,3.5915199915568032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,4,128,1,fp8,fp8,0,2.5780800183614097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,8,128,1,float16,float16,0,3.6901280085245767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,8,128,1,fp8,fp8,0,2.584778626759847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,48,8,128,1,float16,fp8,0,3.6743787129720054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,1,128,1,float16,float16,0,20.62768046061198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,1,128,1,fp8,fp8,0,13.830015818277994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,1,128,1,float16,fp8,0,20.440202077229817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,2,128,1,float16,float16,0,20.298821767171223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,2,128,1,fp8,fp8,0,13.925460815429688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,2,128,1,float16,fp8,0,20.490645090738933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,4,128,1,float16,float16,0,21.118826548258465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,4,128,1,fp8,fp8,0,13.783536275227865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,4,128,1,float16,fp8,0,20.330533345540363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,8,128,1,float16,fp8,0,20.541866302490234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,8,128,1,fp8,fp8,0,14.052448272705078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,48,8,128,1,float16,float16,0,20.64785639444987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,48,128,1,float16,fp8,0,10.633082707722982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,48,128,1,float16,float16,0,10.435365041097006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,1,128,1,float16,fp8,0,10.242048263549805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,1,128,1,fp8,fp8,0,6.93229866027832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,1,128,1,float16,float16,0,10.202005386352539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,2,128,1,float16,float16,0,10.363242467244467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,2,128,1,fp8,fp8,0,6.944608052571614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,2,128,1,float16,fp8,0,10.474133173624674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,4,128,1,float16,fp8,0,10.44217618306478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,4,128,1,float16,float16,0,10.318549474080404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,48,128,1,fp8,fp8,0,7.115333557128906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,4,128,1,fp8,fp8,0,6.918485641479492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,8,128,1,float16,float16,0,10.47441037495931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,8,128,1,fp8,fp8,0,7.144949595133464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,48,8,128,1,float16,fp8,0,10.45351473490397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,1,128,1,float16,float16,0,5.123183886210124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,1,128,1,float16,fp8,0,4.911162694295247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,1,128,1,fp8,fp8,0,3.512048085530599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,2,128,1,float16,float16,0,5.1285654703776045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,48,128,1,float16,float16,0,5.029130617777507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,48,128,1,float16,fp8,0,5.205615997314453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,48,128,1,fp8,fp8,0,3.594325383504232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,2,128,1,float16,fp8,0,5.1127573649088545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,2,128,1,fp8,fp8,0,3.504938761393229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,4,128,1,float16,float16,0,5.278047879536946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,4,128,1,float16,fp8,0,5.238842646280925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,4,128,1,fp8,fp8,0,3.5152587890625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,8,128,1,float16,float16,0,5.140517234802246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,8,128,1,float16,fp8,0,5.0618133544921875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,48,8,128,1,fp8,fp8,0,3.524362564086914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,48,128,1,float16,float16,0,2.6442294120788574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,1,128,1,float16,float16,0,2.6262666384379068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,48,128,1,float16,fp8,0,2.6664746602376304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,48,128,1,fp8,fp8,0,1.9295306205749512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,1,128,1,fp8,fp8,0,1.8924959500630696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,1,128,1,float16,fp8,0,2.5935413042704263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,2,128,1,float16,fp8,0,2.603445370992025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,2,128,1,fp8,fp8,0,1.8919520378112793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,2,128,1,float16,float16,0,2.634789307912191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,4,128,1,float16,float16,0,2.6785173416137695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,4,128,1,float16,fp8,0,2.6120427449544272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,4,128,1,fp8,fp8,0,1.897439956665039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,8,128,1,float16,float16,0,2.6331040064493814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,8,128,1,float16,fp8,0,2.6094239552815757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,48,8,128,1,fp8,fp8,0,1.9025707244873047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,1,128,1,float16,fp8,0,27.545115152994793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,1,128,1,fp8,fp8,0,18.474693298339844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,2,128,1,float16,float16,0,27.13714599609375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,2,128,1,float16,fp8,0,26.937301635742188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,2,128,1,fp8,fp8,0,18.41003163655599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,4,128,1,float16,float16,0,27.255104064941406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,1,128,1,float16,float16,0,26.91747283935547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,4,128,1,fp8,fp8,0,18.52220280965169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,4,128,1,float16,fp8,0,27.605377197265625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,8,128,1,float16,fp8,0,27.097076416015625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,8,128,1,fp8,fp8,0,18.74401092529297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,48,8,128,1,float16,float16,0,27.15887451171875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,48,128,1,float16,float16,0,13.834730784098307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,1,128,1,float16,float16,0,13.940223693847656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,1,128,1,float16,fp8,0,13.542293548583984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,48,128,1,float16,fp8,0,13.591531117757162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,1,128,1,fp8,fp8,0,9.194239934285482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,2,128,1,float16,float16,0,13.591791788736979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,2,128,1,float16,fp8,0,13.6779416402181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,2,128,1,fp8,fp8,0,9.49288558959961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,4,128,1,float16,float16,0,13.595999399820963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,4,128,1,fp8,fp8,0,9.416197458902994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,4,128,1,float16,fp8,0,13.434666951497396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,8,128,1,float16,float16,0,13.630794525146484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,48,128,1,fp8,fp8,0,9.638341267903646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,8,128,1,float16,fp8,0,13.666730244954428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,48,8,128,1,fp8,fp8,0,9.506266911824545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,1,128,1,float16,float16,0,6.6609757741292315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,1,128,1,float16,fp8,0,6.7605438232421875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,1,128,1,fp8,fp8,0,4.6167252858479815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,48,128,1,float16,float16,0,6.812234878540039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,2,128,1,float16,float16,0,6.900992075602214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,48,128,1,float16,fp8,0,6.967861175537109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,48,128,1,fp8,fp8,0,4.768714586893718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,2,128,1,fp8,fp8,0,4.593056042989095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,2,128,1,float16,fp8,0,6.7482452392578125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,4,128,1,float16,float16,0,6.773375829060872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,4,128,1,float16,fp8,0,6.624613444010417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,4,128,1,fp8,fp8,0,4.732570648193359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,8,128,1,float16,fp8,0,6.678693135579427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,8,128,1,float16,float16,0,6.762666702270508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,1,128,1,float16,float16,0,3.37992000579834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,48,8,128,1,fp8,fp8,0,4.645776112874349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,48,128,1,float16,float16,0,3.335285186767578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,48,128,1,float16,fp8,0,3.3943732579549155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,1,128,1,fp8,fp8,0,2.394602616628011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,48,128,1,fp8,fp8,0,2.4836907386779785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,1,128,1,float16,fp8,0,3.2729012171427407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,2,128,1,float16,fp8,0,3.2958027521769204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,2,128,1,float16,float16,0,3.385408083597819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,2,128,1,fp8,fp8,0,2.399061361948649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,4,128,1,float16,float16,0,3.326218605041504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,4,128,1,float16,fp8,0,3.2917439142862954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,4,128,1,fp8,fp8,0,2.4060212771097818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,8,128,1,float16,float16,0,3.3261868158976235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,48,128,1,float16,float16,0,1.778890609741211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,8,128,1,float16,fp8,0,3.3634719848632812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,48,8,128,1,fp8,fp8,0,2.413093407948812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,1,128,1,float16,float16,0,1.7853439648946126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,48,128,1,float16,fp8,0,1.7686079343159993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,1,128,1,float16,fp8,0,1.7575519879659016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,48,128,1,fp8,fp8,0,1.3513545989990234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,1,128,1,fp8,fp8,0,1.313098669052124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,2,128,1,fp8,fp8,0,1.31932266553243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,2,128,1,float16,float16,0,1.792415936787923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,2,128,1,float16,fp8,0,1.7871306737263997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,4,128,1,float16,float16,0,1.7858079274495442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,4,128,1,float16,fp8,0,1.7595680554707844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,4,128,1,fp8,fp8,0,1.3160426616668701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,8,128,1,float16,fp8,0,1.7666559219360352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,8,128,1,float16,float16,0,1.7863839467366536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,48,8,128,1,fp8,fp8,0,1.3220000267028809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,1,128,1,float16,float16,0,16.392597198486328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,1,128,1,fp8,fp8,0,11.04086939493815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,1,128,1,float16,fp8,0,15.922101338704428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,2,128,1,float16,fp8,0,15.911701202392578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,2,128,1,fp8,fp8,0,11.088661193847656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,2,128,1,float16,float16,0,16.190677642822266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,4,128,1,float16,float16,0,16.168997446695965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,4,128,1,float16,fp8,0,16.269930521647137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,4,128,1,fp8,fp8,0,11.095675150553385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,8,128,1,float16,fp8,0,15.948922475179037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,8,128,1,fp8,fp8,0,11.196683247884115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,48,8,128,1,float16,float16,0,16.28448486328125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,48,128,1,float16,float16,0,8.261045455932617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,1,128,1,float16,float16,0,7.909946441650391
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,48,128,1,float16,fp8,0,8.188175837198893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,1,128,1,float16,fp8,0,8.000448226928711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,1,128,1,fp8,fp8,0,5.551792144775391
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,2,128,1,float16,float16,0,8.08560562133789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,2,128,1,float16,fp8,0,7.991482416788737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,2,128,1,fp8,fp8,0,5.524981180826823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,4,128,1,float16,float16,0,8.072901407877604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,4,128,1,float16,fp8,0,7.958970387776692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,48,128,1,fp8,fp8,0,5.857760111490886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,4,128,1,fp8,fp8,0,5.547349294026692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,8,128,1,float16,float16,0,8.060250600179037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,8,128,1,float16,fp8,0,7.850474675496419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,48,8,128,1,fp8,fp8,0,5.589109420776367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,1,128,1,float16,float16,0,3.8333705266316733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,48,128,1,float16,float16,0,3.96724796295166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,1,128,1,float16,fp8,0,3.8701438903808594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,48,128,1,float16,fp8,0,4.036101341247559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,1,128,1,fp8,fp8,0,2.8224480946858725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,2,128,1,float16,float16,0,3.837792078653971
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,2,128,1,float16,fp8,0,3.7869440714518228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,2,128,1,fp8,fp8,0,2.827983856201172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,4,128,1,float16,float16,0,3.9291839599609375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,48,128,1,fp8,fp8,0,2.9796692530314126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,4,128,1,float16,fp8,0,3.8860479990641275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,4,128,1,fp8,fp8,0,2.8356053034464517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,8,128,1,float16,float16,0,3.897226651509603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,8,128,1,float16,fp8,0,3.8143625259399414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,1,128,1,float16,float16,0,2.0131306648254395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,48,8,128,1,fp8,fp8,0,2.8502985636393228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,48,128,1,float16,float16,0,2.0421066284179688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,1,128,1,fp8,fp8,0,1.4944267272949219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,1,128,1,float16,fp8,0,1.9762986501057942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,48,128,1,float16,fp8,0,2.058314641316732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,48,128,1,fp8,fp8,0,1.5650453567504883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,2,128,1,float16,float16,0,2.014618714650472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,2,128,1,float16,fp8,0,1.9880000750223796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,2,128,1,fp8,fp8,0,1.4963626861572266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,4,128,1,float16,float16,0,2.013493378957113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,4,128,1,fp8,fp8,0,1.4995466868082683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,4,128,1,float16,fp8,0,1.9818399747212727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,8,128,1,float16,float16,0,2.019807974497477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,8,128,1,float16,fp8,0,1.9931999842325847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,48,8,128,1,fp8,fp8,0,1.5091946919759114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,48,128,1,float16,float16,0,1.1068267027537029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,1,128,1,float16,float16,0,1.1002453168233235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,1,128,1,fp8,fp8,0,0.8398079872131348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,48,128,1,float16,fp8,0,1.0907306671142578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,1,128,1,float16,fp8,0,1.0840319792429607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,2,128,1,float16,float16,0,1.1053173542022705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,2,128,1,float16,fp8,0,1.084165334701538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,48,128,1,fp8,fp8,0,0.8671840031941732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,2,128,1,fp8,fp8,0,0.8371413548787435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,4,128,1,float16,float16,0,1.1038080056508381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,4,128,1,float16,fp8,0,1.0862773259480794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,4,128,1,fp8,fp8,0,0.8401760260264078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,8,128,1,float16,float16,0,1.108143965403239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,8,128,1,float16,fp8,0,1.0881760120391846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,48,8,128,1,fp8,fp8,0,0.8422880172729492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,1,128,1,float16,float16,0,15.524522145589193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,2,128,1,float16,float16,0,15.613051096598307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,1,128,1,fp8,fp8,0,11.156992594401041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,2,128,1,fp8,fp8,0,11.220160166422525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,2,128,1,float16,fp8,0,15.530731201171875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,1,128,1,float16,fp8,0,15.43014399210612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,4,128,1,float16,float16,0,15.954570770263672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,4,128,1,fp8,fp8,0,11.237611134847006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,4,128,1,float16,fp8,0,15.306906382242838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,8,128,1,float16,fp8,0,15.683306376139322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,8,128,1,fp8,fp8,0,11.359786987304688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,48,8,128,1,float16,float16,0,15.773221333821615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,48,128,1,float16,float16,0,8.166058858235678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,1,128,1,float16,float16,0,7.742282867431641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,48,128,1,float16,fp8,0,8.11179224650065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,1,128,1,float16,fp8,0,7.903594970703125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,1,128,1,fp8,fp8,0,5.626474380493164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,2,128,1,float16,float16,0,7.701925277709961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,2,128,1,float16,fp8,0,7.498010635375977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,2,128,1,fp8,fp8,0,5.633898417154948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,4,128,1,float16,fp8,0,7.6102720896403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,4,128,1,float16,float16,0,7.949653625488281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,48,128,1,fp8,fp8,0,5.993584314982097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,4,128,1,fp8,fp8,0,5.654421488444011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,8,128,1,float16,float16,0,7.9207197825113935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,1,128,1,float16,float16,0,3.789205233256022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,8,128,1,float16,fp8,0,7.560474395751953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,1,128,1,float16,fp8,0,3.695850690205892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,48,8,128,1,fp8,fp8,0,5.69316291809082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,1,128,1,fp8,fp8,0,2.8431145350138345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,2,128,1,float16,float16,0,3.7290401458740234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,48,128,1,fp8,fp8,0,3.050410588582357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,48,128,1,float16,float16,0,3.928096135457357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,2,128,1,float16,fp8,0,3.7134933471679688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,2,128,1,fp8,fp8,0,2.844736099243164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,48,128,1,float16,fp8,0,3.9481118520100913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,4,128,1,float16,float16,0,3.746293385823568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,4,128,1,float16,fp8,0,3.6693493525187173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,4,128,1,fp8,fp8,0,2.8570772806803384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,8,128,1,float16,float16,0,3.748725255330404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,8,128,1,float16,fp8,0,3.6802241007486978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,48,8,128,1,fp8,fp8,0,2.8783092498779297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,1,128,1,float16,float16,0,1.921456019083659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,48,128,1,float16,float16,0,2.03330659866333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,1,128,1,float16,fp8,0,1.8864587148030598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,48,128,1,float16,fp8,0,1.9809707005818684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,1,128,1,fp8,fp8,0,1.4766133626302083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,2,128,1,float16,float16,0,1.9204479853312175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,2,128,1,float16,fp8,0,1.8895039558410645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,48,128,1,fp8,fp8,0,1.5904000600179036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,4,128,1,float16,float16,0,1.9258240063985188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,2,128,1,fp8,fp8,0,1.4798612594604492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,4,128,1,float16,fp8,0,1.896773338317871
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,4,128,1,fp8,fp8,0,1.4856746991475422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,8,128,1,float16,fp8,0,1.9029866854349773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,8,128,1,float16,float16,0,1.9321920077006023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,48,8,128,1,fp8,fp8,0,1.4959519704182942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,1,128,1,float16,float16,0,1.0262347062428792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,1,128,1,float16,fp8,0,1.0058773358662922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,1,128,1,fp8,fp8,0,0.7987360159556071
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,48,128,1,float16,float16,0,1.0514826774597168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,48,128,1,float16,fp8,0,1.0528159936269124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,48,128,1,fp8,fp8,0,0.8512000242869059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,2,128,1,float16,float16,0,1.0292106469472249
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,2,128,1,float16,fp8,0,1.007855971654256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,2,128,1,fp8,fp8,0,0.8002879619598389
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,4,128,1,float16,float16,0,1.0275413195292156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,4,128,1,float16,fp8,0,1.010650634765625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,8,128,1,float16,float16,0,1.0347466468811035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,8,128,1,float16,fp8,0,1.0129173596700032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,4,128,1,fp8,fp8,0,0.8032639821370443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,48,8,128,1,fp8,fp8,0,0.8078347047170004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,1,128,1,float16,float16,0,0.5842186609903971
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,48,128,1,float16,float16,0,0.5900693337122599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,1,128,1,float16,fp8,0,0.5699146588643392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,48,128,1,float16,fp8,0,0.5794506470362345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,48,128,1,fp8,fp8,0,0.46884799003601074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,2,128,1,float16,float16,0,0.583786686261495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,1,128,1,fp8,fp8,0,0.43880534172058105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,2,128,1,float16,fp8,0,0.571999986966451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,2,128,1,fp8,fp8,0,0.4411253531773885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,4,128,1,float16,float16,0,0.5845119953155518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,4,128,1,float16,fp8,0,0.5706080198287964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,4,128,1,fp8,fp8,0,0.44061867396036786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,8,128,1,float16,float16,0,0.5851893424987793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,8,128,1,float16,fp8,0,0.5747520128885905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,48,8,128,1,fp8,fp8,0,0.4445546468098958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,1,128,1,float16,float16,0,9.375781377156576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,1,128,1,fp8,fp8,0,7.141775767008464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,1,128,1,float16,fp8,0,9.34768549601237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,2,128,1,float16,fp8,0,9.342112223307291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,2,128,1,float16,float16,0,9.49943478902181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,2,128,1,fp8,fp8,0,7.189776102701823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,4,128,1,float16,float16,0,9.676090876261393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,4,128,1,float16,fp8,0,9.384112040201822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,4,128,1,fp8,fp8,0,7.18122673034668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,8,128,1,fp8,fp8,0,7.237087885538737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,8,128,1,float16,fp8,0,9.522133509318033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,48,8,128,1,float16,float16,0,9.547109603881836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,1,128,1,float16,float16,0,4.583050727844238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,48,128,1,float16,float16,0,4.962714513142903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,48,128,1,float16,fp8,0,4.81548277537028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,1,128,1,float16,fp8,0,4.485589345296224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,1,128,1,fp8,fp8,0,3.590847969055176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,2,128,1,float16,float16,0,4.635605176289876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,2,128,1,fp8,fp8,0,3.5976905822753906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,2,128,1,float16,fp8,0,4.456975936889648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,4,128,1,float16,float16,0,4.70362122853597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,48,128,1,fp8,fp8,0,3.8929173151652017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,4,128,1,fp8,fp8,0,3.6116161346435547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,4,128,1,float16,fp8,0,4.473002751668294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,8,128,1,float16,float16,0,4.704922676086426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,1,128,1,float16,fp8,0,2.2608532905578613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,1,128,1,float16,float16,0,2.3126559257507324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,8,128,1,fp8,fp8,0,3.6492799123128257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,48,8,128,1,float16,fp8,0,4.741701443990071
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,48,128,1,float16,float16,0,2.4755892753601074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,48,128,1,float16,fp8,0,2.4179360071818032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,48,128,1,fp8,fp8,0,1.9915307362874348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,1,128,1,fp8,fp8,0,1.8233173688252766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,2,128,1,float16,float16,0,2.310629367828369
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,2,128,1,float16,fp8,0,2.264575958251953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,2,128,1,fp8,fp8,0,1.8269707361857097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,4,128,1,float16,float16,0,2.321829319000244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,4,128,1,float16,fp8,0,2.268730640411377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,4,128,1,fp8,fp8,0,1.8359306653340657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,8,128,1,float16,float16,0,2.3359840710957847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,8,128,1,fp8,fp8,0,1.8499679565429688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,48,8,128,1,float16,fp8,0,2.2826293309529624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,1,128,1,float16,float16,0,1.2047733465830486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,1,128,1,float16,fp8,0,1.178656021753947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,48,128,1,float16,float16,0,1.284325361251831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,1,128,1,fp8,fp8,0,0.9616586367289225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,48,128,1,float16,fp8,0,1.2505066394805908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,2,128,1,float16,float16,0,1.2071680227915447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,48,128,1,fp8,fp8,0,1.0498507022857666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,2,128,1,float16,fp8,0,1.1814453601837158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,2,128,1,fp8,fp8,0,0.9611732959747314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,4,128,1,float16,fp8,0,1.1850346724192302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,4,128,1,float16,float16,0,1.2201920350392659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,4,128,1,fp8,fp8,0,0.9661706288655599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,8,128,1,float16,float16,0,1.2151412963867188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,8,128,1,float16,fp8,0,1.1922986507415771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,48,8,128,1,fp8,fp8,0,0.9749013582865397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,1,128,1,float16,float16,0,0.6557439963022867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,1,128,1,float16,fp8,0,0.6393440167109171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,1,128,1,fp8,fp8,0,0.5288480122884115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,48,128,1,float16,float16,0,0.676256020863851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,2,128,1,float16,float16,0,0.656709353129069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,48,128,1,float16,fp8,0,0.6646933158238729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,48,128,1,fp8,fp8,0,0.5688000122706095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,2,128,1,float16,fp8,0,0.640992005666097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,2,128,1,fp8,fp8,0,0.529141346613566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,4,128,1,float16,float16,0,0.6586560010910034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,4,128,1,float16,fp8,0,0.6426133314768473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,4,128,1,fp8,fp8,0,0.5307360092798868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,8,128,1,float16,float16,0,0.660863995552063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,8,128,1,fp8,fp8,0,0.5345866680145264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,48,8,128,1,float16,fp8,0,0.6482079823811849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,1,128,1,float16,float16,0,0.38097067674001056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,48,128,1,float16,float16,0,0.38835732142130536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,48,128,1,float16,fp8,0,0.3819733460744222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,48,128,1,fp8,fp8,0,0.31941332419713336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,1,128,1,fp8,fp8,0,0.2949440081914266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,2,128,1,float16,fp8,0,0.37215999762217206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,1,128,1,float16,fp8,0,0.37236801783243817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,2,128,1,fp8,fp8,0,0.2956533432006836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,2,128,1,float16,float16,0,0.38253867626190186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,4,128,1,float16,float16,0,0.3839786847432454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,4,128,1,float16,fp8,0,0.3737013339996338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,4,128,1,fp8,fp8,0,0.2967093388239543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,8,128,1,float16,float16,0,0.3845280011494954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,8,128,1,fp8,fp8,0,0.29803733030955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,48,8,128,1,float16,fp8,0,0.3755733172098796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,1,128,1,float16,fp8,0,9.561936060587565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,1,128,1,float16,float16,0,10.133087793986002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,2,128,1,float16,float16,0,9.852810541788736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,2,128,1,float16,fp8,0,9.500965118408203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,2,128,1,fp8,fp8,0,7.961162567138672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,1,128,1,fp8,fp8,0,7.956047693888347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,4,128,1,float16,float16,0,9.972976048787435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,4,128,1,float16,fp8,0,9.715167999267578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,4,128,1,fp8,fp8,0,8.014287948608398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,8,128,1,float16,fp8,0,9.848335901896158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,8,128,1,fp8,fp8,0,8.050282796223959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,48,8,128,1,float16,float16,0,9.98845354715983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,48,128,1,float16,float16,0,5.2911787033081055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,1,128,1,float16,float16,0,4.844170570373535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,1,128,1,float16,fp8,0,4.625119845072429
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,48,128,1,float16,fp8,0,5.059439977010091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,1,128,1,fp8,fp8,0,3.929706573486328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,2,128,1,float16,float16,0,4.805525461832683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,2,128,1,float16,fp8,0,4.6956478754679365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,2,128,1,fp8,fp8,0,3.9405012130737305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,48,128,1,fp8,fp8,0,4.341904004414876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,4,128,1,float16,float16,0,4.830410639444987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,4,128,1,float16,fp8,0,4.6635786692301435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,4,128,1,fp8,fp8,0,3.959829330444336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,8,128,1,float16,float16,0,4.960783958435059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,8,128,1,float16,fp8,0,4.733701388041179
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,1,128,1,float16,float16,0,2.3752586046854653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,1,128,1,float16,fp8,0,2.312133312225342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,48,8,128,1,fp8,fp8,0,3.99726931254069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,48,128,1,float16,float16,0,2.613093376159668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,48,128,1,fp8,fp8,0,2.197744051615397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,1,128,1,fp8,fp8,0,1.9693013827006023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,2,128,1,float16,float16,0,2.3798559506734214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,2,128,1,float16,fp8,0,2.318122704823812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,48,128,1,float16,fp8,0,2.5468799273173013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,2,128,1,fp8,fp8,0,1.974671999613444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,4,128,1,float16,float16,0,2.387119928995768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,4,128,1,float16,fp8,0,2.3275094032287598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,4,128,1,fp8,fp8,0,1.9832746187845867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,8,128,1,float16,float16,0,2.403541405995687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,8,128,1,float16,fp8,0,2.343498706817627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,48,8,128,1,fp8,fp8,0,2.006319999694824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,1,128,1,float16,float16,0,1.2250186602274578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,1,128,1,fp8,fp8,0,1.0178666909535725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,48,128,1,float16,float16,0,1.3488319714864094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,2,128,1,float16,float16,0,1.2279040018717449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,1,128,1,float16,fp8,0,1.1924320062001545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,48,128,1,float16,fp8,0,1.301029364267985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,2,128,1,float16,fp8,0,1.1948959827423096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,48,128,1,fp8,fp8,0,1.1396479606628418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,2,128,1,fp8,fp8,0,1.0187733173370361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,4,128,1,float16,fp8,0,1.198298692703247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,4,128,1,float16,float16,0,1.2386399904886882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,4,128,1,fp8,fp8,0,1.0242079893747966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,8,128,1,float16,float16,0,1.239360014597575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,8,128,1,float16,fp8,0,1.2098666826883953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,48,8,128,1,fp8,fp8,0,1.0395092964172363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,1,128,1,float16,float16,0,0.6489280064900717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,1,128,1,float16,fp8,0,0.6327306826909384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,48,128,1,float16,float16,0,0.6932640075683594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,1,128,1,fp8,fp8,0,0.5457119941711426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,2,128,1,float16,float16,0,0.6544640064239502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,48,128,1,float16,fp8,0,0.6786986986796061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,2,128,1,float16,fp8,0,0.6358986695607504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,2,128,1,fp8,fp8,0,0.5474346478780111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,48,128,1,fp8,fp8,0,0.6056480010350546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,4,128,1,float16,float16,0,0.65229332447052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,4,128,1,float16,fp8,0,0.6368800004323324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,4,128,1,fp8,fp8,0,0.5501279830932617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,8,128,1,float16,float16,0,0.6575573285420736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,8,128,1,float16,fp8,0,0.6409066518147787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,48,8,128,1,fp8,fp8,0,0.5549600124359131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,1,128,1,float16,float16,0,0.3640373150507609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,1,128,1,float16,fp8,0,0.3532480001449585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,48,128,1,float16,fp8,0,0.3725920120875041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,48,128,1,float16,float16,0,0.3821706771850586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,48,128,1,fp8,fp8,0,0.3362133502960205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,1,128,1,fp8,fp8,0,0.29477866490681964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,2,128,1,float16,float16,0,0.36533331871032715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,2,128,1,float16,fp8,0,0.3569226662317912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,4,128,1,float16,float16,0,0.365231990814209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,2,128,1,fp8,fp8,0,0.2965173323949178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,4,128,1,fp8,fp8,0,0.2977280020713806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,8,128,1,float16,float16,0,0.36682132879892987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,8,128,1,float16,fp8,0,0.3582613468170166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,4,128,1,float16,fp8,0,0.35710398356119794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,48,8,128,1,fp8,fp8,0,0.30083199342091876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,48,128,1,float16,float16,0,0.2209013303120931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,1,128,1,float16,float16,0,0.20414932568868002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,48,128,1,float16,fp8,0,0.21602133909861246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,1,128,1,float16,fp8,0,0.19853333632151285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,48,128,1,fp8,fp8,0,0.1943999926249186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,1,128,1,fp8,fp8,0,0.17839467525482178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,2,128,1,float16,float16,0,0.20599999030431113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,2,128,1,float16,fp8,0,0.19871467351913452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,2,128,1,fp8,fp8,0,0.17875200510025024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,4,128,1,float16,float16,0,0.20498132705688477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,4,128,1,float16,fp8,0,0.19909334182739258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,4,128,1,fp8,fp8,0,0.17947200934092203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,8,128,1,fp8,fp8,0,0.18040533860524496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,8,128,1,float16,float16,0,0.20578134059906006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,48,8,128,1,float16,fp8,0,0.20155733823776245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,1,128,1,fp8,fp8,0,5.294997215270996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,2,128,1,float16,float16,0,6.317263921101888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,2,128,1,float16,fp8,0,6.041909535725911
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,1,128,1,float16,float16,0,6.261973063151042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,2,128,1,fp8,fp8,0,5.293312072753906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,1,128,1,float16,fp8,0,6.028218587239583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,4,128,1,float16,float16,0,6.333791732788086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,4,128,1,float16,fp8,0,6.075370788574219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,4,128,1,fp8,fp8,0,5.327029228210449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,8,128,1,float16,float16,0,6.500634511311849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,8,128,1,float16,fp8,0,6.08518918355306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,48,8,128,1,fp8,fp8,0,5.368693033854167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,48,128,1,float16,float16,0,3.4307947158813477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,48,128,1,float16,fp8,0,3.3247734705607095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,1,128,1,float16,float16,0,3.07696533203125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,1,128,1,float16,fp8,0,2.9643147786458335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,1,128,1,fp8,fp8,0,2.6221493085225425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,2,128,1,float16,float16,0,3.098128000895182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,2,128,1,float16,fp8,0,2.9759626388549805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,2,128,1,fp8,fp8,0,2.6339200337727866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,4,128,1,float16,float16,0,3.1073811848958335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,4,128,1,float16,fp8,0,2.9867467880249023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,48,128,1,fp8,fp8,0,2.9627199172973633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,4,128,1,fp8,fp8,0,2.644048055013021
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,8,128,1,float16,float16,0,3.1469386418660483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,8,128,1,float16,fp8,0,3.008154551188151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,1,128,1,float16,float16,0,1.5436320304870605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,48,8,128,1,fp8,fp8,0,2.686415990193685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,48,128,1,float16,float16,0,1.730730692545573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,1,128,1,float16,fp8,0,1.5020853678385417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,1,128,1,fp8,fp8,0,1.3254133065541585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,48,128,1,fp8,fp8,0,1.5083252588907878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,2,128,1,float16,float16,0,1.5486613909403484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,2,128,1,float16,fp8,0,1.5029333432515461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,48,128,1,float16,fp8,0,1.6894879341125488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,2,128,1,fp8,fp8,0,1.3295573393503826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,4,128,1,float16,float16,0,1.5572800636291504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,4,128,1,float16,fp8,0,1.5084479649861653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,4,128,1,fp8,fp8,0,1.3375199635823567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,8,128,1,float16,fp8,0,1.5235519409179688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,8,128,1,float16,float16,0,1.5651787122090657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,48,8,128,1,fp8,fp8,0,1.354101339975993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,1,128,1,float16,float16,0,0.8038132985432943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,48,128,1,float16,float16,0,0.8870453039805094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,1,128,1,float16,fp8,0,0.7805013656616211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,2,128,1,float16,float16,0,0.8065813382466634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,1,128,1,fp8,fp8,0,0.691818634668986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,2,128,1,float16,fp8,0,0.7829866409301758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,48,128,1,float16,fp8,0,0.8723680178324381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,2,128,1,fp8,fp8,0,0.6950293382008871
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,48,128,1,fp8,fp8,0,0.7828213373819987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,4,128,1,float16,float16,0,0.809397300084432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,4,128,1,float16,fp8,0,0.7860053380330404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,4,128,1,fp8,fp8,0,0.697322686513265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,8,128,1,float16,float16,0,0.8150080045064291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,1,128,1,float16,float16,0,0.4323306481043498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,8,128,1,fp8,fp8,0,0.7067626317342123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,48,8,128,1,float16,fp8,0,0.7934559981028239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,48,128,1,float16,float16,0,0.46722133954366046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,48,128,1,float16,fp8,0,0.4570346673329671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,1,128,1,float16,fp8,0,0.42054398854573566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,48,128,1,fp8,fp8,0,0.4228266477584839
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,1,128,1,fp8,fp8,0,0.37459735075632733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,2,128,1,float16,float16,0,0.4350399971008301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,2,128,1,fp8,fp8,0,0.37676799297332764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,2,128,1,float16,fp8,0,0.42160534858703613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,4,128,1,float16,float16,0,0.4352693160374959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,4,128,1,float16,fp8,0,0.4235519965489705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,4,128,1,fp8,fp8,0,0.3798559904098511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,8,128,1,float16,float16,0,0.4379466772079468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,8,128,1,float16,fp8,0,0.4271999994913737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,48,8,128,1,fp8,fp8,0,0.3838026523590088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,1,128,1,float16,float16,0,0.24609067042668661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,48,128,1,float16,fp8,0,0.2555253307024638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,48,128,1,fp8,fp8,0,0.23548267285029092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,48,128,1,float16,float16,0,0.26083733638127643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,1,128,1,fp8,fp8,0,0.20838934183120728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,1,128,1,float16,fp8,0,0.23849600553512573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,2,128,1,float16,float16,0,0.2472426692644755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,2,128,1,float16,fp8,0,0.23906666040420532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,2,128,1,fp8,fp8,0,0.2093813419342041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,4,128,1,float16,float16,0,0.24879467487335205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,4,128,1,fp8,fp8,0,0.20974934101104736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,8,128,1,float16,fp8,0,0.24304000536600748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,8,128,1,float16,float16,0,0.25064533948898315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,8,128,1,fp8,fp8,0,0.2113493283589681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,48,4,128,1,float16,fp8,0,0.242031991481781
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,48,128,1,float16,fp8,0,0.15149333079655966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,48,128,1,float16,float16,0,0.154639999071757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,48,128,1,fp8,fp8,0,0.14119999607404074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,1,128,1,float16,float16,0,0.14218133687973022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,1,128,1,fp8,fp8,0,0.12813867131868997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,1,128,1,float16,fp8,0,0.13873599966367087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,2,128,1,float16,fp8,0,0.13849066694577536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,2,128,1,float16,float16,0,0.1425279974937439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,2,128,1,fp8,fp8,0,0.12863467137018839
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,4,128,1,float16,float16,0,0.1431893308957418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,4,128,1,float16,fp8,0,0.13899200161298117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,4,128,1,fp8,fp8,0,0.12956266601880392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,8,128,1,float16,float16,0,0.14443733294804892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,8,128,1,float16,fp8,0,0.13936533530553183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,48,8,128,1,fp8,fp8,0,0.1309440036614736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,1,128,1,float16,float16,0,6.018389383951823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,1,128,1,fp8,fp8,0,5.199632008870442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,2,128,1,fp8,fp8,0,5.389557520548503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,1,128,1,float16,fp8,0,5.968320210774739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,2,128,1,float16,float16,0,6.08568000793457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,2,128,1,float16,fp8,0,6.01040522257487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,4,128,1,float16,float16,0,6.274074554443359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,4,128,1,float16,fp8,0,6.259024302164714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,8,128,1,float16,float16,0,6.329216003417969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,4,128,1,fp8,fp8,0,5.820597330729167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,8,128,1,float16,fp8,0,6.335642496744792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,48,8,128,1,fp8,fp8,0,5.847877502441406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,48,128,1,float16,float16,0,3.7641067504882812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,1,128,1,float16,float16,0,2.9460426966349282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,48,128,1,float16,fp8,0,3.6667200724283853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,1,128,1,float16,fp8,0,2.9112586975097656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,1,128,1,fp8,fp8,0,2.5972426732381186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,2,128,1,float16,float16,0,3.0036532084147134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,2,128,1,float16,fp8,0,2.919322649637858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,2,128,1,fp8,fp8,0,2.6949440638224282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,4,128,1,float16,fp8,0,3.112959861755371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,4,128,1,fp8,fp8,0,2.9009812672932944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,48,128,1,fp8,fp8,0,3.177493413289388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,4,128,1,float16,float16,0,3.116586685180664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,8,128,1,float16,float16,0,3.139967918395996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,8,128,1,float16,fp8,0,3.1562827428181968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,1,128,1,float16,fp8,0,1.46778138478597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,1,128,1,float16,float16,0,1.4769280751546223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,48,8,128,1,fp8,fp8,0,2.941557248433431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,1,128,1,fp8,fp8,0,1.3017386595408122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,48,128,1,float16,float16,0,1.884335994720459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,48,128,1,float16,fp8,0,1.8145440419514973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,2,128,1,float16,float16,0,1.4850932757059734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,48,128,1,fp8,fp8,0,1.5903679529825847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,2,128,1,fp8,fp8,0,1.350277264912923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,2,128,1,float16,fp8,0,1.473530610402425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,4,128,1,float16,float16,0,1.5532959302266438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,4,128,1,float16,fp8,0,1.5447840690612793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,4,128,1,fp8,fp8,0,1.4652320543924968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,8,128,1,float16,float16,0,1.5650293032328289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,8,128,1,float16,fp8,0,1.587685267130534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,48,8,128,1,fp8,fp8,0,1.479200045267741
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,1,128,1,float16,float16,0,0.7536373138427734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,48,128,1,float16,float16,0,0.9543840090433756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,1,128,1,float16,fp8,0,0.7501973311106364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,1,128,1,fp8,fp8,0,0.6622346639633179
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,2,128,1,float16,float16,0,0.7617812951405843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,48,128,1,float16,fp8,0,0.9131893316904703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,2,128,1,float16,fp8,0,0.7516319751739502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,48,128,1,fp8,fp8,0,0.8047733306884766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,2,128,1,fp8,fp8,0,0.6819732983907064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,4,128,1,float16,float16,0,0.7809546788533529
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,4,128,1,float16,fp8,0,0.7745652993520101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,4,128,1,fp8,fp8,0,0.7445440292358398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,8,128,1,float16,float16,0,0.7834293047587076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,8,128,1,float16,fp8,0,0.787775993347168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,48,8,128,1,fp8,fp8,0,0.7484266757965088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,1,128,1,float16,float16,0,0.3900586764017741
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,1,128,1,float16,fp8,0,0.3895893494288127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,48,128,1,float16,float16,0,0.4886666536331177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,48,128,1,fp8,fp8,0,0.4118293523788452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,48,128,1,float16,fp8,0,0.47200000286102295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,2,128,1,float16,float16,0,0.3928906520207723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,2,128,1,float16,fp8,0,0.3924586772918701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,1,128,1,fp8,fp8,0,0.34230931599934894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,2,128,1,fp8,fp8,0,0.34694401423136395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,4,128,1,fp8,fp8,0,0.37993065516153973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,4,128,1,float16,fp8,0,0.39900267124176025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,4,128,1,float16,float16,0,0.40238932768503827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,8,128,1,float16,float16,0,0.4063946803410848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,8,128,1,fp8,fp8,0,0.38605332374572754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,48,8,128,1,float16,fp8,0,0.40674134095509845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,1,128,1,float16,float16,0,0.21187732617060342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,48,128,1,float16,float16,0,0.25994133949279785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,1,128,1,float16,fp8,0,0.21188799540201822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,1,128,1,fp8,fp8,0,0.17671465873718262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,2,128,1,float16,float16,0,0.2113920052846273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,48,128,1,fp8,fp8,0,0.2207039992014567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,48,128,1,float16,fp8,0,0.2516053318977356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,2,128,1,fp8,fp8,0,0.17712533473968506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,2,128,1,float16,fp8,0,0.21057599782943726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,4,128,1,float16,float16,0,0.2160266637802124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,4,128,1,float16,fp8,0,0.21568000316619873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,4,128,1,fp8,fp8,0,0.19317332903544107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,8,128,1,float16,float16,0,0.2209813396135966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,8,128,1,float16,fp8,0,0.21655466159184775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,1,128,1,float16,float16,0,0.1139413317044576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,48,128,1,float16,float16,0,0.1534773310025533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,48,128,1,float16,fp8,0,0.14890133341153464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,1,128,1,fp8,fp8,0,0.0969546635945638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,48,128,1,fp8,fp8,0,0.11884267131487529
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,1,128,1,float16,fp8,0,0.11401599645614624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,48,8,128,1,fp8,fp8,0,0.19693867365519205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,2,128,1,float16,float16,0,0.11395733555157979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,2,128,1,fp8,fp8,0,0.09797333677609761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,2,128,1,float16,fp8,0,0.11427733302116394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,4,128,1,float16,float16,0,0.11463466286659241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,4,128,1,float16,fp8,0,0.11593600114186604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,4,128,1,fp8,fp8,0,0.10382399956385295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,8,128,1,float16,float16,0,0.11614400148391724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,8,128,1,float16,fp8,0,0.11629333098729451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,48,8,128,1,fp8,fp8,0,0.10569066802660625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,1,128,1,float16,float16,0,0.06298133234182994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,1,128,1,float16,fp8,0,0.06270933151245117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,48,128,1,float16,fp8,0,0.0738560010989507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,48,128,1,fp8,fp8,0,0.07035199801127116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,48,128,1,float16,float16,0,0.07531733314196269
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,1,128,1,fp8,fp8,0,0.05624000231424967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,2,128,1,float16,float16,0,0.06319466729958852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,2,128,1,fp8,fp8,0,0.05705599983533224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,4,128,1,float16,float16,0,0.06418133278687795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,4,128,1,float16,fp8,0,0.06514133512973785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,2,128,1,float16,fp8,0,0.06351466476917267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,8,128,1,float16,float16,0,0.06543999910354614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,4,128,1,fp8,fp8,0,0.06118933359781901
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,8,128,1,float16,fp8,0,0.06531733274459839
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,48,8,128,1,fp8,fp8,0,0.06176533301671346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,1,128,1,float16,float16,0,4.774282773335774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,1,128,1,float16,fp8,0,4.599402745564778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,2,128,1,fp8,fp8,0,4.471722602844238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,1,128,1,fp8,fp8,0,4.31716791788737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,4,128,1,float16,float16,0,4.94543997446696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,2,128,1,float16,float16,0,4.777039845784505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,2,128,1,float16,fp8,0,4.689663887023926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,4,128,1,fp8,fp8,0,4.934650739034017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,4,128,1,float16,fp8,0,5.016874631245931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,8,128,1,float16,float16,0,5.021391868591309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,8,128,1,float16,fp8,0,5.037253379821777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,48,8,128,1,fp8,fp8,0,4.979999860127767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,48,128,1,float16,float16,0,3.139589309692383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,1,128,1,float16,float16,0,2.294655958811442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,1,128,1,float16,fp8,0,2.260709285736084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,48,128,1,float16,fp8,0,3.00875186920166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,1,128,1,fp8,fp8,0,2.158890724182129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,2,128,1,float16,float16,0,2.3624213536580405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,2,128,1,float16,fp8,0,2.2772533098856607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,2,128,1,fp8,fp8,0,2.241157372792562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,4,128,1,float16,float16,0,2.463749408721924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,48,128,1,fp8,fp8,0,2.7350826263427734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,4,128,1,float16,fp8,0,2.492191950480143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,4,128,1,fp8,fp8,0,2.456021308898926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,8,128,1,float16,float16,0,2.5052266120910645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,1,128,1,float16,float16,0,1.1579360167185466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,8,128,1,float16,fp8,0,2.5173439979553223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,48,8,128,1,fp8,fp8,0,2.484970728556315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,48,128,1,float16,float16,0,1.5782186190287273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,48,128,1,float16,fp8,0,1.508021354675293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,1,128,1,fp8,fp8,0,1.0699626604715984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,1,128,1,float16,fp8,0,1.1438079675038655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,2,128,1,float16,float16,0,1.1646506786346436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,2,128,1,float16,fp8,0,1.1479520003000896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,48,128,1,fp8,fp8,0,1.3680000305175781
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,2,128,1,fp8,fp8,0,1.1203733285268147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,4,128,1,float16,float16,0,1.2322773138682048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,4,128,1,float16,fp8,0,1.2211573123931885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,4,128,1,fp8,fp8,0,1.2274879614512126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,8,128,1,float16,float16,0,1.2438186804453533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,8,128,1,float16,fp8,0,1.2524853547414143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,48,8,128,1,fp8,fp8,0,1.243712027867635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,1,128,1,float16,float16,0,0.5892959833145142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,48,128,1,float16,float16,0,0.7912639776865641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,1,128,1,fp8,fp8,0,0.5449866851170858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,1,128,1,float16,fp8,0,0.5845439831415812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,2,128,1,float16,float16,0,0.5933653513590494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,48,128,1,float16,fp8,0,0.7583519617716471
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,2,128,1,float16,fp8,0,0.5858293374379476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,48,128,1,fp8,fp8,0,0.6876107056935629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,2,128,1,fp8,fp8,0,0.5676053365071615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,4,128,1,float16,float16,0,0.6118026574452718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,4,128,1,float16,fp8,0,0.6093653440475464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,4,128,1,fp8,fp8,0,0.6287466684977213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,8,128,1,float16,float16,0,0.6173119942347208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,8,128,1,float16,fp8,0,0.6191840171813965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,48,8,128,1,fp8,fp8,0,0.6344319979349772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,1,128,1,float16,float16,0,0.30374399820963544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,1,128,1,float16,fp8,0,0.30325865745544434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,1,128,1,fp8,fp8,0,0.28246400753657025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,48,128,1,float16,float16,0,0.4103253285090129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,48,128,1,float16,fp8,0,0.3928426504135132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,2,128,1,float16,float16,0,0.30849599838256836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,48,128,1,fp8,fp8,0,0.35186131795247394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,2,128,1,float16,fp8,0,0.3045919934908549
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,2,128,1,fp8,fp8,0,0.2868480086326599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,4,128,1,float16,float16,0,0.31691733996073407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,4,128,1,fp8,fp8,0,0.3190559943517049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,4,128,1,float16,fp8,0,0.3139946659406026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,8,128,1,float16,float16,0,0.3211146593093872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,8,128,1,fp8,fp8,0,0.32517866293589276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,1,128,1,float16,float16,0,0.16411200165748596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,48,128,1,float16,float16,0,0.2175040046374003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,1,128,1,float16,fp8,0,0.1653439998626709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,48,8,128,1,float16,fp8,0,0.31803733110427856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,48,128,1,float16,fp8,0,0.20934400955835977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,48,128,1,fp8,fp8,0,0.18944533665974936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,1,128,1,fp8,fp8,0,0.15037332971890768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,2,128,1,float16,float16,0,0.16554133097330728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,2,128,1,float16,fp8,0,0.16522133350372314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,2,128,1,fp8,fp8,0,0.15154666701952615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,4,128,1,float16,fp8,0,0.1691360076268514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,4,128,1,fp8,fp8,0,0.16825600465138754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,4,128,1,float16,float16,0,0.17004267374674478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,8,128,1,float16,fp8,0,0.1720586617787679
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,8,128,1,fp8,fp8,0,0.17096533377965292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,48,8,128,1,float16,float16,0,0.17430400848388672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,48,128,1,float16,float16,0,0.13338667154312134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,48,128,1,float16,fp8,0,0.12945066889127096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,1,128,1,float16,float16,0,0.09331199526786804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,48,128,1,fp8,fp8,0,0.10219200452168782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,1,128,1,fp8,fp8,0,0.08441600203514099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,2,128,1,float16,fp8,0,0.0936959981918335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,1,128,1,float16,fp8,0,0.09259200096130371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,2,128,1,fp8,fp8,0,0.08416533470153809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,2,128,1,float16,float16,0,0.0922879974047343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,4,128,1,float16,fp8,0,0.09548266728719075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,4,128,1,float16,float16,0,0.0951039989789327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,4,128,1,fp8,fp8,0,0.09097599983215332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,8,128,1,float16,float16,0,0.09554133812586467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,8,128,1,float16,fp8,0,0.09523199995358785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,48,8,128,1,fp8,fp8,0,0.09262933333714803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,48,128,1,float16,fp8,0,0.06454400221506755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,1,128,1,float16,float16,0,0.05271466573079427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,48,128,1,fp8,fp8,0,0.06070399781068166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,1,128,1,float16,fp8,0,0.05301333467165629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,1,128,1,fp8,fp8,0,0.04828266799449921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,48,128,1,float16,float16,0,0.06729066868623097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,2,128,1,float16,float16,0,0.053445334235827126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,2,128,1,float16,fp8,0,0.05299733579158783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,2,128,1,fp8,fp8,0,0.04933866858482361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,4,128,1,float16,float16,0,0.055120001236597695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,4,128,1,float16,fp8,0,0.055162668228149414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,4,128,1,fp8,fp8,0,0.05319466690222422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,8,128,1,float16,fp8,0,0.05566933254400889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,8,128,1,fp8,fp8,0,0.05429333448410034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,48,8,128,1,float16,float16,0,0.055231998364130654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,48,128,1,float16,float16,0,0.03965866565704346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,48,128,1,float16,fp8,0,0.03866666555404663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,48,128,1,fp8,fp8,0,0.037647999823093414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,1,128,1,float16,float16,0,0.03401600072781245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,1,128,1,float16,fp8,0,0.03443733354409536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,1,128,1,fp8,fp8,0,0.03306666761636734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,2,128,1,float16,fp8,0,0.034160000582536064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,4,128,1,float16,float16,0,0.03488533447186152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,4,128,1,float16,fp8,0,0.03500800083080927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,2,128,1,fp8,fp8,0,0.033439998825391136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,2,128,1,float16,float16,0,0.034517332911491394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,4,128,1,fp8,fp8,0,0.0349440003434817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,8,128,1,float16,float16,0,0.0355679988861084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,8,128,1,float16,fp8,0,0.03572800010442734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,48,8,128,1,fp8,fp8,0,0.0349440003434817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,1,128,1,float16,float16,0,1.9820586840311687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,2,128,1,fp8,fp8,0,1.9626026153564453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,1,128,1,fp8,fp8,0,1.9025440216064453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,2,128,1,float16,float16,0,2.0446240107218423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,4,128,1,float16,float16,0,2.149178663889567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,2,128,1,float16,fp8,0,1.9641173680623372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,1,128,1,float16,fp8,0,1.952298641204834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,4,128,1,float16,fp8,0,2.1376585960388184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,4,128,1,fp8,fp8,0,2.187877337137858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,8,128,1,float16,fp8,0,2.198911984761556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,8,128,1,float16,float16,0,2.1816320419311523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,48,8,128,1,fp8,fp8,0,2.218224048614502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,48,128,1,float16,float16,0,1.4345866839090984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,1,128,1,float16,float16,0,0.9923040072123209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,48,128,1,float16,fp8,0,1.3611733118693035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,1,128,1,float16,fp8,0,0.9859626293182373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,2,128,1,float16,float16,0,1.00382399559021
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,1,128,1,fp8,fp8,0,0.9323733647664388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,2,128,1,float16,fp8,0,0.9923946857452393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,2,128,1,fp8,fp8,0,0.9835626284281412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,48,128,1,fp8,fp8,0,1.24673064549764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,4,128,1,float16,float16,0,1.0665813287099202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,4,128,1,float16,fp8,0,1.0515626271565754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,4,128,1,fp8,fp8,0,1.09280530611674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,8,128,1,float16,float16,0,1.0842026869455974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,8,128,1,float16,fp8,0,1.0933067003885906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,1,128,1,float16,float16,0,0.5076959927876791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,48,8,128,1,fp8,fp8,0,1.1133386294047039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,1,128,1,fp8,fp8,0,0.47679467995961505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,48,128,1,float16,float16,0,0.7278079986572266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,1,128,1,float16,fp8,0,0.5032800038655599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,2,128,1,float16,float16,0,0.513205329577128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,48,128,1,fp8,fp8,0,0.6256373325983683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,2,128,1,float16,fp8,0,0.5081760088602701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,48,128,1,float16,fp8,0,0.6957706610361735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,2,128,1,fp8,fp8,0,0.5002559820810953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,4,128,1,float16,float16,0,0.5341279904047648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,4,128,1,fp8,fp8,0,0.5602560043334961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,4,128,1,float16,fp8,0,0.528005321820577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,8,128,1,float16,float16,0,0.5354453325271606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,8,128,1,float16,fp8,0,0.5444373289744059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,48,8,128,1,fp8,fp8,0,0.5668266614278158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,1,128,1,float16,fp8,0,0.26340266068776447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,1,128,1,float16,float16,0,0.26316799720128375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,48,128,1,float16,float16,0,0.37540801366170246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,1,128,1,fp8,fp8,0,0.24939199288686117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,48,128,1,float16,fp8,0,0.35766398906707764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,48,128,1,fp8,fp8,0,0.3189706603686015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,2,128,1,float16,float16,0,0.26501866181691486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,2,128,1,float16,fp8,0,0.263754665851593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,2,128,1,fp8,fp8,0,0.2539093295733134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,4,128,1,float16,float16,0,0.2741546630859375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,4,128,1,float16,fp8,0,0.2735733389854431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,4,128,1,fp8,fp8,0,0.2867093284924825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,8,128,1,float16,float16,0,0.2787359952926636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,8,128,1,fp8,fp8,0,0.2914133270581563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,48,8,128,1,float16,fp8,0,0.27749866247177124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,1,128,1,float16,float16,0,0.14320533474286398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,1,128,1,float16,fp8,0,0.14378666877746582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,48,128,1,float16,fp8,0,0.19154133399327597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,48,128,1,float16,float16,0,0.1986186703046163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,1,128,1,fp8,fp8,0,0.13477333386739096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,48,128,1,fp8,fp8,0,0.1725119948387146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,2,128,1,float16,float16,0,0.1438826620578766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,2,128,1,float16,fp8,0,0.14353600144386292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,2,128,1,fp8,fp8,0,0.13736533125241598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,4,128,1,float16,float16,0,0.14834133783976236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,4,128,1,fp8,fp8,0,0.1518239974975586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,8,128,1,float16,fp8,0,0.15067199865976968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,8,128,1,float16,float16,0,0.15270400047302246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,8,128,1,fp8,fp8,0,0.15642133355140686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,48,4,128,1,float16,fp8,0,0.14686399698257446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,1,128,1,float16,float16,0,0.08317333459854126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,1,128,1,float16,fp8,0,0.08273066580295563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,48,128,1,float16,float16,0,0.12236266334851582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,1,128,1,fp8,fp8,0,0.07517866790294647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,48,128,1,fp8,fp8,0,0.09549867113431294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,48,128,1,float16,fp8,0,0.11801600456237793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,2,128,1,float16,float16,0,0.08307733138402303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,2,128,1,float16,fp8,0,0.08358400066693623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,4,128,1,float16,float16,0,0.0857973297437032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,4,128,1,float16,fp8,0,0.08612799644470215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,2,128,1,fp8,fp8,0,0.07666666805744171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,4,128,1,fp8,fp8,0,0.08175466458002727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,8,128,1,float16,float16,0,0.08616532882054646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,1,128,1,float16,float16,0,0.04797333478927612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,1,128,1,float16,fp8,0,0.04780800143877665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,8,128,1,fp8,fp8,0,0.08480532964070638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,48,8,128,1,float16,fp8,0,0.08646933237711589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,48,128,1,float16,float16,0,0.06452266871929169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,48,128,1,float16,fp8,0,0.06146133442719778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,48,128,1,fp8,fp8,0,0.057333335280418396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,1,128,1,fp8,fp8,0,0.043136000633239746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,2,128,1,float16,fp8,0,0.04850133260091146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,2,128,1,float16,float16,0,0.04809066653251648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,4,128,1,fp8,fp8,0,0.04763199885686239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,2,128,1,fp8,fp8,0,0.04320000112056732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,4,128,1,float16,float16,0,0.05030933519204458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,4,128,1,float16,fp8,0,0.04985066751639048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,8,128,1,float16,float16,0,0.050479998191197716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,8,128,1,float16,fp8,0,0.05016533533732096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,48,128,1,float16,float16,0,0.03752533346414566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,48,8,128,1,fp8,fp8,0,0.04851733148097992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,1,128,1,float16,fp8,0,0.03329599897066752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,48,128,1,fp8,fp8,0,0.03487999985615412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,1,128,1,float16,float16,0,0.03249600032965342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,2,128,1,float16,float16,0,0.03321066747109095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,1,128,1,fp8,fp8,0,0.030224000414212544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,2,128,1,float16,fp8,0,0.033413333197434746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,48,128,1,float16,fp8,0,0.036981334288915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,2,128,1,fp8,fp8,0,0.03072533259789149
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,4,128,1,float16,float16,0,0.03358400116364161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,4,128,1,float16,fp8,0,0.03368533402681351
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,4,128,1,fp8,fp8,0,0.032229334115982056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,8,128,1,float16,float16,0,0.03436266630887985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,8,128,1,float16,fp8,0,0.0345920001467069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,48,128,1,float16,float16,0,0.026682667434215546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,1,128,1,float16,float16,0,0.024533333877722423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,1,128,1,float16,fp8,0,0.02497600018978119
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,48,128,1,float16,fp8,0,0.0262719988822937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,48,128,1,fp8,fp8,0,0.02499199906984965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,48,8,128,1,fp8,fp8,0,0.032560000816980995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,2,128,1,float16,float16,0,0.024933333198229473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,1,128,1,fp8,fp8,0,0.02277333289384842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,2,128,1,float16,fp8,0,0.025077333052953083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,2,128,1,fp8,fp8,0,0.023056000471115112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,4,128,1,float16,fp8,0,0.025290665527184803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,4,128,1,float16,float16,0,0.024885334074497223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,8,128,1,float16,float16,0,0.024821333587169647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,4,128,1,fp8,fp8,0,0.024005333582560223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,8,128,1,fp8,fp8,0,0.024085332949956257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,48,8,128,1,float16,fp8,0,0.025279998779296875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,1,128,1,float16,fp8,0,0.9047626654307047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,1,128,1,float16,float16,0,0.9088266690572103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,1,128,1,fp8,fp8,0,0.9453439712524414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,2,128,1,float16,float16,0,0.9411946932474772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,2,128,1,float16,fp8,0,0.9136746724446615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,2,128,1,fp8,fp8,0,0.9739092985788981
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,4,128,1,float16,float16,0,1.0001599788665771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,4,128,1,fp8,fp8,0,1.0909333229064941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,4,128,1,float16,fp8,0,0.9803520043691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,8,128,1,float16,fp8,0,1.009504000345866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,8,128,1,float16,float16,0,1.020693302154541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,48,128,1,float16,float16,0,0.7486666838328043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,48,8,128,1,fp8,fp8,0,1.1141280333201091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,1,128,1,float16,float16,0,0.4682399829228719
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,1,128,1,float16,fp8,0,0.4652693271636963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,1,128,1,fp8,fp8,0,0.4737279812494914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,2,128,1,float16,float16,0,0.47811198234558105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,48,128,1,float16,fp8,0,0.7170186837514242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,2,128,1,float16,fp8,0,0.47200532754262287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,48,128,1,fp8,fp8,0,0.6233173211415609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,2,128,1,fp8,fp8,0,0.49245866139729816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,4,128,1,float16,float16,0,0.5023359855016073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,4,128,1,float16,fp8,0,0.4947520097096761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,4,128,1,fp8,fp8,0,0.5560160080591837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,8,128,1,float16,float16,0,0.5045066674550375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,8,128,1,float16,fp8,0,0.5025386810302734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,1,128,1,float16,float16,0,0.24626133839289346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,48,8,128,1,fp8,fp8,0,0.5654346545537313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,48,128,1,float16,float16,0,0.38307201862335205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,1,128,1,float16,fp8,0,0.24546132485071817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,48,128,1,fp8,fp8,0,0.31947733958562213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,48,128,1,float16,fp8,0,0.3686399857203166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,1,128,1,fp8,fp8,0,0.24649600187937418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,2,128,1,float16,fp8,0,0.24934399127960205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,2,128,1,float16,float16,0,0.25069334109624225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,2,128,1,fp8,fp8,0,0.25252266724904376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,4,128,1,float16,float16,0,0.26123199860254925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,4,128,1,fp8,fp8,0,0.2852426568667094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,4,128,1,float16,fp8,0,0.2587786714235942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,8,128,1,float16,fp8,0,0.26154667139053345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,8,128,1,fp8,fp8,0,0.2929546634356181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,48,8,128,1,float16,float16,0,0.26341332991917926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,48,128,1,float16,float16,0,0.2079626719156901
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,1,128,1,float16,float16,0,0.1369439959526062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,48,128,1,float16,fp8,0,0.1972586711247762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,1,128,1,float16,fp8,0,0.1360106666882833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,48,128,1,fp8,fp8,0,0.16837332646052042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,1,128,1,fp8,fp8,0,0.13588266571362814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,2,128,1,float16,fp8,0,0.13780799508094788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,2,128,1,float16,float16,0,0.13820800185203552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,2,128,1,fp8,fp8,0,0.13569066921869913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,4,128,1,float16,fp8,0,0.14333867033322653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,4,128,1,float16,float16,0,0.14323733250300089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,4,128,1,fp8,fp8,0,0.15017599860827127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,8,128,1,float16,fp8,0,0.14462400476137796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,8,128,1,float16,float16,0,0.14636266231536865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,48,8,128,1,fp8,fp8,0,0.15437333782513937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,1,128,1,float16,float16,0,0.07788800199826558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,48,128,1,float16,float16,0,0.11300800244013469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,1,128,1,float16,fp8,0,0.07724800209204356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,1,128,1,fp8,fp8,0,0.07736533383528392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,48,128,1,float16,fp8,0,0.11011733611424764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,48,128,1,fp8,fp8,0,0.09435733159383138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,2,128,1,float16,fp8,0,0.07832533121109009
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,2,128,1,fp8,fp8,0,0.07600000003973643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,4,128,1,float16,float16,0,0.08089066545168559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,2,128,1,float16,float16,0,0.07804266611735027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,4,128,1,float16,fp8,0,0.08052266637484233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,8,128,1,float16,float16,0,0.08264000217119853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,4,128,1,fp8,fp8,0,0.08228266735871632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,8,128,1,float16,fp8,0,0.08242666721343994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,1,128,1,float16,float16,0,0.045312002301216125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,48,8,128,1,fp8,fp8,0,0.08425600330034892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,48,128,1,float16,float16,0,0.06211733321348826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,1,128,1,float16,fp8,0,0.04557866851488749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,48,128,1,fp8,fp8,0,0.05657599866390228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,48,128,1,float16,fp8,0,0.059205333391825356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,1,128,1,fp8,fp8,0,0.042709335684776306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,2,128,1,float16,float16,0,0.04572799801826477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,2,128,1,float16,fp8,0,0.046367997924486794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,2,128,1,fp8,fp8,0,0.043552001317342125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,4,128,1,float16,float16,0,0.0476800004641215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,4,128,1,float16,fp8,0,0.047456001242001854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,8,128,1,float16,float16,0,0.047882666190465294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,4,128,1,fp8,fp8,0,0.047226667404174805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,8,128,1,float16,fp8,0,0.04751466711362203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,48,128,1,float16,float16,0,0.03559466699759165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,48,128,1,float16,fp8,0,0.03559466699759165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,48,128,1,fp8,fp8,0,0.0348693331082662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,48,8,128,1,fp8,fp8,0,0.05119466781616211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,1,128,1,float16,float16,0,0.03209600100914637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,1,128,1,fp8,fp8,0,0.03009066730737686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,2,128,1,fp8,fp8,0,0.030784000953038532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,2,128,1,float16,fp8,0,0.03229333211978277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,2,128,1,float16,float16,0,0.032458665470282234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,4,128,1,float16,float16,0,0.033402666449546814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,1,128,1,float16,fp8,0,0.03493333359559377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,4,128,1,float16,fp8,0,0.033359999457995095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,4,128,1,fp8,fp8,0,0.03216533362865448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,8,128,1,float16,float16,0,0.03281066566705704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,8,128,1,float16,fp8,0,0.03366933266321818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,48,8,128,1,fp8,fp8,0,0.032042667269706726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,48,128,1,float16,float16,0,0.02595199892918269
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,48,128,1,fp8,fp8,0,0.02526933451493581
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,48,128,1,float16,fp8,0,0.02532800038655599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,1,128,1,float16,float16,0,0.023904000719388325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,1,128,1,float16,fp8,0,0.024314666787783306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,1,128,1,fp8,fp8,0,0.02248000105222066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,2,128,1,float16,float16,0,0.02350933353106181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,2,128,1,float16,fp8,0,0.023743999501069386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,4,128,1,float16,fp8,0,0.02492266645034154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,2,128,1,fp8,fp8,0,0.023130667706330616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,4,128,1,float16,float16,0,0.024506665766239166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,4,128,1,fp8,fp8,0,0.024346667031447094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,8,128,1,float16,float16,0,0.02426133304834366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,8,128,1,float16,fp8,0,0.024197332561016083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,48,8,128,1,fp8,fp8,0,0.023557332654794056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,48,128,1,float16,float16,0,0.02033599962790807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,1,128,1,float16,float16,0,0.020576000213623047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,48,128,1,float16,fp8,0,0.020330666253964107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,48,128,1,fp8,fp8,0,0.02057066683967908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,1,128,1,fp8,fp8,0,0.020021333048741024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,1,128,1,float16,fp8,0,0.020128000527620316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,2,128,1,float16,float16,0,0.019999999552965164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,2,128,1,fp8,fp8,0,0.02041600023706754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,2,128,1,float16,fp8,0,0.020362666497627895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,4,128,1,float16,float16,0,0.020021333048741024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,4,128,1,fp8,fp8,0,0.020309332758188248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,4,128,1,float16,fp8,0,0.02042666698495547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,8,128,1,float16,float16,0,0.02024000013868014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,8,128,1,float16,fp8,0,0.020261333634455998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,48,8,128,1,fp8,fp8,0,0.020421333611011505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,1,128,1,float16,float16,0,0.5807413260142008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,1,128,1,float16,fp8,0,0.581061323483785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,2,128,1,float16,fp8,0,0.5829759836196899
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,1,128,1,fp8,fp8,0,0.6282506783803304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,2,128,1,fp8,fp8,0,0.6485493183135986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,2,128,1,float16,float16,0,0.588757316271464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,4,128,1,float16,float16,0,0.6168746550877889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,4,128,1,float16,fp8,0,0.6104640165964762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,8,128,1,float16,float16,0,0.6227680047353109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,4,128,1,fp8,fp8,0,0.7189973195393881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,8,128,1,float16,fp8,0,0.6184533437093099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,48,8,128,1,fp8,fp8,0,0.7320106824239095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,48,128,1,float16,fp8,0,0.3719946543375651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,48,128,1,float16,float16,0,0.38681066036224365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,1,128,1,float16,float16,0,0.29892800251642865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,1,128,1,float16,fp8,0,0.2995520035425822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,1,128,1,fp8,fp8,0,0.32466665903727215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,2,128,1,float16,float16,0,0.302842656771342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,2,128,1,float16,fp8,0,0.30166399478912354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,48,128,1,fp8,fp8,0,0.3977280060450236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,2,128,1,fp8,fp8,0,0.33322133620580036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,4,128,1,float16,float16,0,0.31493866443634033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,4,128,1,float16,fp8,0,0.31143999099731445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,4,128,1,fp8,fp8,0,0.3648639917373657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,8,128,1,float16,float16,0,0.3179733355840047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,8,128,1,float16,fp8,0,0.31462933619817096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,48,8,128,1,fp8,fp8,0,0.3724000056584676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,1,128,1,float16,float16,0,0.16023466984430948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,1,128,1,float16,fp8,0,0.16049066185951233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,48,128,1,float16,fp8,0,0.1972106695175171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,1,128,1,fp8,fp8,0,0.1745599905649821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,48,128,1,fp8,fp8,0,0.20696000258127847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,48,128,1,float16,float16,0,0.20403732856114706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,2,128,1,float16,fp8,0,0.16057067116101584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,2,128,1,float16,float16,0,0.16103466351826987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,2,128,1,fp8,fp8,0,0.1760853330294291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,4,128,1,float16,float16,0,0.16715733210245767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,4,128,1,float16,fp8,0,0.1665013333161672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,4,128,1,fp8,fp8,0,0.1901386578877767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,8,128,1,float16,fp8,0,0.168229341506958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,8,128,1,float16,float16,0,0.16976000865300497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,1,128,1,float16,float16,0,0.08806932965914409
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,48,128,1,float16,fp8,0,0.10636267066001892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,48,128,1,float16,float16,0,0.11034133036931355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,48,128,1,fp8,fp8,0,0.11407466729482015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,48,8,128,1,fp8,fp8,0,0.1948266625404358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,1,128,1,float16,fp8,0,0.08842133482297261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,2,128,1,float16,float16,0,0.08715732892354329
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,1,128,1,fp8,fp8,0,0.09493866562843323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,2,128,1,float16,fp8,0,0.08773866295814514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,2,128,1,fp8,fp8,0,0.09591466188430786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,4,128,1,float16,float16,0,0.09076799949010213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,4,128,1,fp8,fp8,0,0.10257599751154582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,8,128,1,float16,float16,0,0.0913813312848409
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,4,128,1,float16,fp8,0,0.09082667032877605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,8,128,1,fp8,fp8,0,0.1048906644185384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,48,128,1,float16,float16,0,0.06188266475995382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,48,128,1,float16,fp8,0,0.06091199815273285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,48,8,128,1,float16,fp8,0,0.0918986697991689
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,48,128,1,fp8,fp8,0,0.06612800061702728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,1,128,1,float16,float16,0,0.05045333504676819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,1,128,1,float16,fp8,0,0.050944000482559204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,1,128,1,fp8,fp8,0,0.05384533107280731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,2,128,1,float16,float16,0,0.05156266689300537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,2,128,1,float16,fp8,0,0.051072001457214355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,4,128,1,float16,float16,0,0.05274133384227753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,8,128,1,float16,float16,0,0.05366933345794678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,4,128,1,float16,fp8,0,0.05271466573079427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,2,128,1,fp8,fp8,0,0.05429866909980774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,4,128,1,fp8,fp8,0,0.05898666878541311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,8,128,1,float16,fp8,0,0.05357866485913595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,48,8,128,1,fp8,fp8,0,0.06035733222961426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,48,128,1,float16,float16,0,0.03469333300987879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,48,128,1,float16,fp8,0,0.03417599946260452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,48,128,1,fp8,fp8,0,0.03875733415285746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,1,128,1,float16,fp8,0,0.03290133426586787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,1,128,1,fp8,fp8,0,0.034287999073664345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,2,128,1,float16,float16,0,0.03286933402220408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,1,128,1,float16,float16,0,0.03266133368015289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,2,128,1,float16,fp8,0,0.03278400003910065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,2,128,1,fp8,fp8,0,0.03422400106986364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,4,128,1,float16,float16,0,0.03380800038576126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,8,128,1,float16,float16,0,0.033520000676314034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,4,128,1,fp8,fp8,0,0.03606933355331421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,8,128,1,fp8,fp8,0,0.035631999373435974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,48,128,1,float16,float16,0,0.027237333357334137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,4,128,1,float16,fp8,0,0.033941333492596946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,48,128,1,float16,fp8,0,0.02735466758410136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,48,8,128,1,float16,fp8,0,0.03404266635576884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,48,128,1,fp8,fp8,0,0.028938665986061096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,1,128,1,float16,float16,0,0.02643200010061264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,1,128,1,float16,fp8,0,0.02621866762638092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,2,128,1,float16,fp8,0,0.026565333207448322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,1,128,1,fp8,fp8,0,0.02658133457104365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,2,128,1,float16,float16,0,0.025850666066010792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,8,128,1,float16,float16,0,0.026288000245889027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,4,128,1,float16,float16,0,0.02696000039577484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,4,128,1,float16,fp8,0,0.026677332818508148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,4,128,1,fp8,fp8,0,0.02773866554101308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,2,128,1,fp8,fp8,0,0.027104000250498455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,8,128,1,float16,fp8,0,0.026672000686327618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,48,128,1,float16,float16,0,0.020010666300853092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,48,128,1,float16,fp8,0,0.020586666961510975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,48,128,1,fp8,fp8,0,0.020821332931518555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,1,128,1,fp8,fp8,0,0.019802667200565338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,1,128,1,float16,float16,0,0.01893866683046023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,48,8,128,1,fp8,fp8,0,0.027477333943049114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,1,128,1,float16,fp8,0,0.01937599976857503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,2,128,1,float16,float16,0,0.018613333503405254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,4,128,1,float16,float16,0,0.019317333896954853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,2,128,1,fp8,fp8,0,0.019834666202465694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,2,128,1,float16,fp8,0,0.01971199984351794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,4,128,1,float16,fp8,0,0.019653332730134327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,8,128,1,float16,fp8,0,0.019248000035683315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,8,128,1,fp8,fp8,0,0.020186666399240494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,48,128,1,float16,float16,0,0.018383999665578205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,4,128,1,fp8,fp8,0,0.020154666155576706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,48,128,1,fp8,fp8,0,0.01937599976857503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,48,128,1,float16,fp8,0,0.01899733394384384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,48,8,128,1,float16,float16,0,0.019226666539907455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,1,128,1,float16,fp8,0,0.01876266673207283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,1,128,1,float16,float16,0,0.018112000077962875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,1,128,1,fp8,fp8,0,0.019285333653291065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,2,128,1,float16,fp8,0,0.018645333747069042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,4,128,1,float16,fp8,0,0.01858666663368543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,2,128,1,fp8,fp8,0,0.01951466624935468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,4,128,1,fp8,fp8,0,0.019039999693632126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,2,128,1,float16,float16,0,0.018266666680574417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,4,128,1,float16,float16,0,0.018058666338523228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,8,128,1,float16,float16,0,0.01859733338157336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,8,128,1,fp8,fp8,0,0.01924266666173935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,48,8,128,1,float16,fp8,0,0.018570666511853535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,1,128,1,float16,float16,0,0.39391998449961346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,1,128,1,float16,fp8,0,0.39474666118621826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,2,128,1,fp8,fp8,0,0.5076533158620199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,2,128,1,float16,fp8,0,0.4001013437906901
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,4,128,1,float16,float16,0,0.4148533344268799
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,2,128,1,float16,float16,0,0.40009065469106037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,1,128,1,fp8,fp8,0,0.4990239938100179
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,4,128,1,float16,fp8,0,0.41433600584665936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,4,128,1,fp8,fp8,0,0.5469280083974203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,8,128,1,float16,float16,0,0.422762672106425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,8,128,1,float16,fp8,0,0.42210666338602704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,48,8,128,1,fp8,fp8,0,0.5516320069630941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,1,128,1,float16,float16,0,0.20559465885162354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,48,128,1,float16,float16,0,0.2516160011291504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,48,128,1,float16,fp8,0,0.24333866437276205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,1,128,1,float16,fp8,0,0.20662933588027954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,1,128,1,fp8,fp8,0,0.2624906698862712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,48,128,1,fp8,fp8,0,0.2952959934870402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,2,128,1,float16,float16,0,0.2092533310254415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,2,128,1,float16,fp8,0,0.2091360092163086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,4,128,1,float16,fp8,0,0.2153493364651998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,4,128,1,fp8,fp8,0,0.27853866418202716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,2,128,1,fp8,fp8,0,0.2641013264656067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,8,128,1,float16,float16,0,0.22181334098180136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,4,128,1,float16,float16,0,0.21638399362564087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,8,128,1,float16,fp8,0,0.21965332825978598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,48,8,128,1,fp8,fp8,0,0.28221333026885986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,1,128,1,float16,float16,0,0.10873599847157796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,48,128,1,float16,float16,0,0.1333440045515696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,1,128,1,fp8,fp8,0,0.14281599720319113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,48,128,1,float16,fp8,0,0.12888532876968384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,1,128,1,float16,fp8,0,0.1090186635653178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,48,128,1,fp8,fp8,0,0.15796266992886862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,2,128,1,float16,float16,0,0.10863999525705974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,2,128,1,float16,fp8,0,0.10941333572069804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,2,128,1,fp8,fp8,0,0.1425279974937439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,4,128,1,float16,float16,0,0.11043733358383179
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,4,128,1,float16,fp8,0,0.11181867122650146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,8,128,1,float16,fp8,0,0.11358933647473653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,4,128,1,fp8,fp8,0,0.14916800459225973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,8,128,1,float16,float16,0,0.11379733681678772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,48,128,1,float16,float16,0,0.07406400144100189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,48,8,128,1,fp8,fp8,0,0.1524853308995565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,48,128,1,float16,fp8,0,0.07116266588370006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,48,128,1,fp8,fp8,0,0.08872532844543457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,1,128,1,float16,float16,0,0.06062933305899302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,2,128,1,float16,float16,0,0.06147199869155884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,1,128,1,float16,fp8,0,0.06084266801675161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,2,128,1,fp8,fp8,0,0.0792799989382426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,2,128,1,float16,fp8,0,0.06133866806825002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,1,128,1,fp8,fp8,0,0.07769066592057546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,4,128,1,float16,float16,0,0.06284800171852112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,4,128,1,fp8,fp8,0,0.08190399905045827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,4,128,1,float16,fp8,0,0.0628959983587265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,8,128,1,float16,fp8,0,0.0639573335647583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,8,128,1,fp8,fp8,0,0.084197332461675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,48,8,128,1,float16,float16,0,0.06356800099213918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,48,128,1,float16,float16,0,0.03976533313592275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,48,128,1,float16,fp8,0,0.039605334401130676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,1,128,1,float16,float16,0,0.03724266588687897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,1,128,1,float16,fp8,0,0.037461332976818085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,48,128,1,fp8,fp8,0,0.04926399886608124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,1,128,1,fp8,fp8,0,0.04593066871166229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,2,128,1,float16,float16,0,0.03754133234421412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,2,128,1,fp8,fp8,0,0.04674666623274485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,4,128,1,float16,float16,0,0.03835200021664301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,4,128,1,float16,fp8,0,0.038560000558694206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,4,128,1,fp8,fp8,0,0.0481333335240682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,2,128,1,float16,fp8,0,0.038106667498747505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,8,128,1,float16,float16,0,0.038346665600935616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,8,128,1,float16,fp8,0,0.038346665600935616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,48,8,128,1,fp8,fp8,0,0.04850666721661886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,48,128,1,float16,float16,0,0.028592000404993694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,1,128,1,float16,fp8,0,0.026608000199000042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,48,128,1,float16,fp8,0,0.02812800059715907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,48,128,1,fp8,fp8,0,0.033189333975315094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,1,128,1,float16,float16,0,0.026357332865397137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,1,128,1,fp8,fp8,0,0.030794667700926464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,2,128,1,fp8,fp8,0,0.03160533308982849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,2,128,1,float16,fp8,0,0.0266239990790685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,4,128,1,float16,fp8,0,0.027141332626342773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,2,128,1,float16,float16,0,0.026687999566396076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,4,128,1,float16,float16,0,0.02657066782315572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,8,128,1,float16,fp8,0,0.027280000348885853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,8,128,1,fp8,fp8,0,0.03182933231194814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,48,128,1,float16,float16,0,0.021925332645575207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,4,128,1,fp8,fp8,0,0.03161066770553589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,48,128,1,float16,fp8,0,0.021781332790851593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,48,128,1,fp8,fp8,0,0.0245919997493426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,48,8,128,1,float16,float16,0,0.027221334477265675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,1,128,1,float16,float16,0,0.021850667893886566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,1,128,1,float16,fp8,0,0.020997333029905956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,1,128,1,fp8,fp8,0,0.023930666347344715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,4,128,1,float16,fp8,0,0.021920000513394673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,2,128,1,float16,float16,0,0.02160533269246419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,2,128,1,float16,fp8,0,0.021701333423455555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,4,128,1,fp8,fp8,0,0.023973333338896435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,8,128,1,float16,float16,0,0.021642667551835377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,4,128,1,float16,float16,0,0.02195200075705846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,2,128,1,fp8,fp8,0,0.02385066697994868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,8,128,1,float16,fp8,0,0.022053333620230358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,48,128,1,float16,fp8,0,0.017994667092959087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,48,128,1,float16,float16,0,0.01777600000301997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,1,128,1,float16,float16,0,0.017792000124851864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,1,128,1,fp8,fp8,0,0.01930133377512296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,1,128,1,float16,fp8,0,0.017690667261679966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,48,8,128,1,fp8,fp8,0,0.02386133372783661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,48,128,1,fp8,fp8,0,0.020154666155576706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,2,128,1,float16,float16,0,0.017557332913080852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,2,128,1,float16,fp8,0,0.01786133274435997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,2,128,1,fp8,fp8,0,0.019402666638294857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,4,128,1,float16,float16,0,0.017466666797796886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,4,128,1,float16,fp8,0,0.018053332964579265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,8,128,1,float16,fp8,0,0.0179626668492953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,4,128,1,fp8,fp8,0,0.019120000302791595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,8,128,1,fp8,fp8,0,0.019850666324297588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,48,128,1,float16,fp8,0,0.01736533393462499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,48,128,1,float16,float16,0,0.016773333152135212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,48,8,128,1,float16,float16,0,0.01762666677435239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,1,128,1,float16,float16,0,0.016965333372354507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,48,128,1,fp8,fp8,0,0.018842666099468868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,1,128,1,float16,fp8,0,0.017477333545684814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,1,128,1,fp8,fp8,0,0.019029332945744198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,2,128,1,float16,float16,0,0.017360000560681026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,4,128,1,float16,float16,0,0.017386666188637417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,2,128,1,float16,fp8,0,0.01740266631046931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,4,128,1,float16,fp8,0,0.01725333308180173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,4,128,1,fp8,fp8,0,0.018533332894245785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,2,128,1,fp8,fp8,0,0.018288000176350277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,8,128,1,float16,fp8,0,0.01800000046690305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,8,128,1,float16,float16,0,0.01754666616519292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,48,8,128,1,fp8,fp8,0,0.0185759998857975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,1,128,1,float16,fp8,0,0.33458133538564044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,1,128,1,fp8,fp8,0,0.4392373164494832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,1,128,1,float16,float16,0,0.33527998129526776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,4,128,1,float16,float16,0,0.3422880172729492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,2,128,1,float16,fp8,0,0.3372693459192912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,2,128,1,fp8,fp8,0,0.4423893292744954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,2,128,1,float16,float16,0,0.3373173475265503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,4,128,1,float16,fp8,0,0.343509316444397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,8,128,1,float16,fp8,0,0.3471466700236003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,8,128,1,fp8,fp8,0,0.46324801445007324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,4,128,1,fp8,fp8,0,0.4562400182088216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,48,128,1,float16,float16,0,0.18618667125701904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,48,8,128,1,float16,float16,0,0.34794668356577557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,1,128,1,float16,float16,0,0.17273066441218057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,48,128,1,float16,fp8,0,0.1828533411026001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,48,128,1,fp8,fp8,0,0.24655999739964804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,1,128,1,float16,fp8,0,0.17252800861994425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,2,128,1,float16,float16,0,0.1729066570599874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,1,128,1,fp8,fp8,0,0.23211199045181274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,2,128,1,float16,fp8,0,0.17302932341893515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,2,128,1,fp8,fp8,0,0.2323519984881083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,4,128,1,float16,float16,0,0.17596266667048135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,4,128,1,float16,fp8,0,0.17570134003957114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,4,128,1,fp8,fp8,0,0.23957866430282593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,8,128,1,float16,float16,0,0.17814399798711142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,8,128,1,fp8,fp8,0,0.24136000871658325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,48,128,1,float16,float16,0,0.10045866171518962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,48,8,128,1,float16,fp8,0,0.17828800280888876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,1,128,1,float16,float16,0,0.0934933324654897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,48,128,1,float16,fp8,0,0.09843200445175171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,48,128,1,fp8,fp8,0,0.1355946660041809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,1,128,1,float16,fp8,0,0.09392000238100688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,1,128,1,fp8,fp8,0,0.12496532996495564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,2,128,1,float16,float16,0,0.09416000048319499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,4,128,1,float16,float16,0,0.095551997423172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,2,128,1,fp8,fp8,0,0.12585066755612692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,4,128,1,float16,fp8,0,0.0956053336461385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,4,128,1,fp8,fp8,0,0.12940800189971924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,2,128,1,float16,fp8,0,0.09356799721717834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,8,128,1,float16,float16,0,0.09610666831334432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,8,128,1,float16,fp8,0,0.09646933277448018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,48,8,128,1,fp8,fp8,0,0.12980799873669943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,48,128,1,float16,fp8,0,0.05374933282534281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,48,128,1,float16,float16,0,0.05378133555253347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,48,128,1,fp8,fp8,0,0.07330133517583211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,1,128,1,float16,fp8,0,0.05515733361244202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,1,128,1,fp8,fp8,0,0.07001600166161855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,1,128,1,float16,float16,0,0.053802669048309326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,2,128,1,float16,float16,0,0.053690666953722634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,2,128,1,float16,fp8,0,0.05400000015894572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,4,128,1,float16,float16,0,0.055306668082873024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,2,128,1,fp8,fp8,0,0.07030400137106578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,4,128,1,float16,fp8,0,0.055685331424077354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,4,128,1,fp8,fp8,0,0.0715946654478709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,8,128,1,float16,fp8,0,0.05482666691144308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,8,128,1,fp8,fp8,0,0.0717493345340093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,48,128,1,float16,float16,0,0.03568000098069509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,48,128,1,float16,fp8,0,0.03536533315976461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,48,8,128,1,float16,float16,0,0.054325332244237266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,1,128,1,float16,float16,0,0.03568000098069509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,1,128,1,float16,fp8,0,0.0360000009338061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,1,128,1,fp8,fp8,0,0.04364799956480662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,2,128,1,float16,float16,0,0.036357333262761436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,2,128,1,float16,fp8,0,0.03626666714747747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,48,128,1,fp8,fp8,0,0.044026667873064675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,4,128,1,float16,float16,0,0.03634133438269297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,4,128,1,float16,fp8,0,0.03621866554021835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,4,128,1,fp8,fp8,0,0.04422933359940847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,8,128,1,float16,float16,0,0.03696000079313914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,8,128,1,float16,fp8,0,0.03678400069475174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,2,128,1,fp8,fp8,0,0.04358933369318644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,48,128,1,float16,float16,0,0.024458666642506916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,48,128,1,float16,fp8,0,0.02498133232196172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,48,8,128,1,fp8,fp8,0,0.04468800127506256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,48,128,1,fp8,fp8,0,0.02958400050799052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,1,128,1,float16,float16,0,0.023898666103680927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,1,128,1,float16,fp8,0,0.02442666639884313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,2,128,1,float16,float16,0,0.024090667565663654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,2,128,1,float16,fp8,0,0.02481599897146225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,1,128,1,fp8,fp8,0,0.028192001084486645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,2,128,1,fp8,fp8,0,0.028389332195123036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,4,128,1,float16,float16,0,0.024138666689395905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,4,128,1,float16,fp8,0,0.024383999407291412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,4,128,1,fp8,fp8,0,0.029253333806991577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,8,128,1,float16,float16,0,0.024192000428835552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,8,128,1,float16,fp8,0,0.02462399999300639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,48,128,1,float16,fp8,0,0.020645332833131153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,48,128,1,fp8,fp8,0,0.023728000621000927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,1,128,1,float16,float16,0,0.020314666132132213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,48,128,1,float16,float16,0,0.020245333512624104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,1,128,1,float16,fp8,0,0.020687999824682873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,48,8,128,1,fp8,fp8,0,0.0284853329261144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,1,128,1,fp8,fp8,0,0.023562667270501454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,2,128,1,float16,float16,0,0.020138667275508244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,2,128,1,float16,fp8,0,0.020661332954963047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,2,128,1,fp8,fp8,0,0.023306667804718018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,4,128,1,float16,fp8,0,0.020597333709398907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,4,128,1,float16,float16,0,0.020773333807786305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,8,128,1,float16,float16,0,0.02075733368595441
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,4,128,1,fp8,fp8,0,0.023365333676338196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,8,128,1,float16,fp8,0,0.020938667158285778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,48,8,128,1,fp8,fp8,0,0.023925334215164185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,48,128,1,float16,fp8,0,0.017797333498795826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,48,128,1,float16,float16,0,0.017301333447297413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,48,128,1,fp8,fp8,0,0.019760000209013622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,1,128,1,float16,float16,0,0.017055999487638474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,2,128,1,float16,fp8,0,0.017386666188637417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,1,128,1,fp8,fp8,0,0.018837332725524902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,2,128,1,float16,float16,0,0.017184000462293625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,2,128,1,fp8,fp8,0,0.01899733394384384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,1,128,1,float16,fp8,0,0.017045332739750545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,4,128,1,float16,fp8,0,0.017312000195185345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,4,128,1,float16,float16,0,0.01743999992807706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,4,128,1,fp8,fp8,0,0.018725333114465077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,8,128,1,float16,float16,0,0.016602666427691776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,8,128,1,float16,fp8,0,0.017749333133300144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,48,8,128,1,fp8,fp8,0,0.019487999379634857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,48,128,1,float16,fp8,0,0.016885332763195038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,48,128,1,fp8,fp8,0,0.018885333091020584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,1,128,1,float16,float16,0,0.016682667036851246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,48,128,1,float16,float16,0,0.016250666230916977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,1,128,1,float16,fp8,0,0.016970666746298473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,1,128,1,fp8,fp8,0,0.01887999971707662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,2,128,1,float16,float16,0,0.016976000120242436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,2,128,1,float16,fp8,0,0.017263999829689663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,4,128,1,float16,float16,0,0.016789333273967106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,2,128,1,fp8,fp8,0,0.018687999496857326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,4,128,1,fp8,fp8,0,0.01859733338157336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,8,128,1,float16,float16,0,0.017173333714405697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,8,128,1,fp8,fp8,0,0.01869333287080129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,4,128,1,float16,fp8,0,0.01720533271630605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,48,8,128,1,float16,fp8,0,0.01757866640885671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,1,128,1,fp8,fp8,0,27.44744110107422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,1,128,1,float16,float16,0,42.04810078938802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,1,128,1,float16,fp8,0,41.683441162109375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,2,128,1,float16,float16,0,41.572837829589844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,2,128,1,fp8,fp8,0,28.102528889973957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,2,128,1,float16,fp8,0,41.8974609375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,4,128,1,float16,float16,0,41.933545430501304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,4,128,1,float16,fp8,0,41.623888651529946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,4,128,1,fp8,fp8,0,27.60027821858724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,8,128,1,float16,float16,0,42.394795735677086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,8,128,1,fp8,fp8,0,28.324607849121094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,40,8,128,1,float16,fp8,0,41.22967020670573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,40,128,1,float16,float16,0,21.12141291300456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,1,128,1,float16,float16,0,21.15222422281901
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,40,128,1,float16,fp8,0,21.067712148030598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,1,128,1,float16,fp8,0,20.88699213663737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,1,128,1,fp8,fp8,0,14.1189816792806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,2,128,1,float16,fp8,0,20.87190882364909
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,2,128,1,float16,float16,0,20.93835194905599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,40,128,1,fp8,fp8,0,13.99832026163737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,2,128,1,fp8,fp8,0,13.94601567586263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,4,128,1,float16,fp8,0,21.044843037923176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,4,128,1,float16,float16,0,20.975391387939453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,4,128,1,fp8,fp8,0,14.200815836588541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,8,128,1,float16,fp8,0,20.74786122639974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,8,128,1,float16,float16,0,21.11953608194987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,1,128,1,float16,float16,0,10.840970357259115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,1,128,1,float16,fp8,0,10.483280181884766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,40,8,128,1,fp8,fp8,0,13.892400105794271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,1,128,1,fp8,fp8,0,7.005002975463867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,40,128,1,float16,float16,0,10.672565460205078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,40,128,1,float16,fp8,0,10.44694964090983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,40,128,1,fp8,fp8,0,6.967418670654297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,2,128,1,fp8,fp8,0,6.901664098103841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,2,128,1,float16,fp8,0,10.605557123819986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,2,128,1,float16,float16,0,10.8809445699056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,4,128,1,float16,float16,0,10.755648295084635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,4,128,1,float16,fp8,0,10.64629872639974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,4,128,1,fp8,fp8,0,6.97433598836263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,8,128,1,float16,float16,0,10.706731160481771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,8,128,1,float16,fp8,0,10.72173817952474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,1,128,1,float16,float16,0,5.451119740804036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,40,8,128,1,fp8,fp8,0,6.861888249715169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,40,128,1,float16,fp8,0,5.2945601145426435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,40,128,1,fp8,fp8,0,3.576298713684082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,40,128,1,float16,float16,0,5.470448176066081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,1,128,1,float16,fp8,0,5.151285489400228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,1,128,1,fp8,fp8,0,3.5594666798909507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,2,128,1,float16,float16,0,5.419471740722656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,2,128,1,float16,fp8,0,5.293951988220215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,2,128,1,fp8,fp8,0,3.553370793660482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,4,128,1,float16,float16,0,5.4268747965494795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,4,128,1,float16,fp8,0,5.434272130330403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,4,128,1,fp8,fp8,0,3.558143933614095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,8,128,1,float16,float16,0,5.43829345703125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,8,128,1,fp8,fp8,0,3.571471850077311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,40,8,128,1,float16,fp8,0,5.321173350016276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,1,128,1,float16,float16,0,24.17388153076172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,1,128,1,float16,fp8,0,24.272720336914062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,1,128,1,fp8,fp8,0,16.54523213704427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,2,128,1,float16,float16,0,24.173973083496094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,2,128,1,fp8,fp8,0,16.029749552408855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,2,128,1,float16,fp8,0,24.11608123779297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,4,128,1,float16,float16,0,24.33356221516927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,4,128,1,float16,fp8,0,24.144187927246094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,4,128,1,fp8,fp8,0,16.232789357503254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,8,128,1,float16,float16,0,24.89240010579427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,8,128,1,float16,fp8,0,24.2281010945638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,40,8,128,1,fp8,fp8,0,16.180260976155598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,40,128,1,float16,float16,0,12.240837097167969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,40,128,1,float16,fp8,0,12.12490717569987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,1,128,1,float16,float16,0,12.291941324869791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,1,128,1,float16,fp8,0,12.299120585123697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,1,128,1,fp8,fp8,0,8.07693354288737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,2,128,1,float16,float16,0,12.096602121988932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,2,128,1,float16,fp8,0,12.09225591023763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,2,128,1,fp8,fp8,0,8.051087697347006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,4,128,1,float16,float16,0,12.334181467692057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,40,128,1,fp8,fp8,0,8.219631830851236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,4,128,1,fp8,fp8,0,8.054762522379557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,4,128,1,float16,fp8,0,12.370895385742188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,8,128,1,float16,float16,0,12.147098541259766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,8,128,1,float16,fp8,0,12.11895497639974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,40,8,128,1,fp8,fp8,0,8.194618860880533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,40,128,1,float16,float16,0,6.272645314534505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,1,128,1,float16,float16,0,6.201690673828125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,40,128,1,float16,fp8,0,5.949525197347005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,1,128,1,float16,fp8,0,5.912725448608398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,1,128,1,fp8,fp8,0,4.029589335123698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,2,128,1,float16,float16,0,6.113375981648763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,2,128,1,float16,fp8,0,6.114501317342122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,2,128,1,fp8,fp8,0,4.039434750874837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,40,128,1,fp8,fp8,0,4.110645294189453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,4,128,1,float16,float16,0,6.092346827189128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,4,128,1,fp8,fp8,0,4.038485209147136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,4,128,1,float16,fp8,0,6.081434885660808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,8,128,1,float16,float16,0,6.0135040283203125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,1,128,1,float16,float16,0,3.0532054901123047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,8,128,1,float16,fp8,0,6.0675309499104815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,40,8,128,1,fp8,fp8,0,4.064597447713216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,40,128,1,float16,float16,0,3.0764694213867188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,1,128,1,float16,fp8,0,3.0285011927286782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,1,128,1,fp8,fp8,0,2.163621266682943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,40,128,1,float16,fp8,0,3.0499998728434243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,40,128,1,fp8,fp8,0,2.1922453244527182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,2,128,1,float16,float16,0,3.065530776977539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,2,128,1,float16,fp8,0,3.137941360473633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,2,128,1,fp8,fp8,0,2.162160078684489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,4,128,1,float16,float16,0,3.0685386657714844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,4,128,1,float16,fp8,0,3.0166187286376953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,4,128,1,fp8,fp8,0,2.1644746462504068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,8,128,1,float16,fp8,0,3.029402732849121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,8,128,1,float16,float16,0,3.066645304361979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,40,8,128,1,fp8,fp8,0,2.1723626454671225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,1,128,1,float16,float16,0,17.25717290242513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,1,128,1,float16,fp8,0,17.162986755371094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,1,128,1,fp8,fp8,0,11.533087412516275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,2,128,1,fp8,fp8,0,11.532986958821615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,2,128,1,float16,float16,0,17.06667709350586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,2,128,1,float16,fp8,0,17.26251729329427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,4,128,1,float16,float16,0,17.666656494140625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,4,128,1,fp8,fp8,0,11.587871551513672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,4,128,1,float16,fp8,0,17.0916748046875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,8,128,1,float16,float16,0,17.183509826660156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,8,128,1,float16,fp8,0,17.135648091634113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,40,128,1,float16,float16,0,8.812527974446615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,40,8,128,1,fp8,fp8,0,11.594980875651041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,1,128,1,float16,float16,0,8.666346867879232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,1,128,1,float16,fp8,0,8.458762486775717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,40,128,1,float16,fp8,0,8.720975875854492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,1,128,1,fp8,fp8,0,5.661637624104817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,2,128,1,float16,float16,0,8.564170837402344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,2,128,1,float16,fp8,0,8.620320002237955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,2,128,1,fp8,fp8,0,5.75167973836263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,40,128,1,fp8,fp8,0,5.973066965738933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,4,128,1,float16,float16,0,8.460528055826822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,4,128,1,fp8,fp8,0,5.79310925801595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,4,128,1,float16,fp8,0,8.702831904093424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,8,128,1,float16,float16,0,8.816090901692709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,8,128,1,float16,fp8,0,8.667754491170248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,40,8,128,1,fp8,fp8,0,5.8654130299886065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,1,128,1,float16,float16,0,4.40016523996989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,40,128,1,float16,float16,0,4.36461861928304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,1,128,1,float16,fp8,0,4.21340815226237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,1,128,1,fp8,fp8,0,2.9331092834472656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,40,128,1,float16,fp8,0,4.281909306844075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,40,128,1,fp8,fp8,0,3.0078452428181968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,2,128,1,float16,float16,0,4.267392158508301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,2,128,1,float16,fp8,0,4.323653221130371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,2,128,1,fp8,fp8,0,2.935530662536621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,4,128,1,float16,float16,0,4.156629244486491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,4,128,1,float16,fp8,0,4.197717348734538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,4,128,1,fp8,fp8,0,2.9462080001831055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,8,128,1,float16,float16,0,4.153210639953613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,8,128,1,float16,fp8,0,4.13161055246989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,40,8,128,1,fp8,fp8,0,2.9567254384358725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,40,128,1,float16,float16,0,2.217696030934652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,40,128,1,float16,fp8,0,2.1843786239624023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,1,128,1,float16,float16,0,2.208672046661377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,1,128,1,float16,fp8,0,2.1782827377319336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,40,128,1,fp8,fp8,0,1.6244640350341797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,1,128,1,fp8,fp8,0,1.5896320343017578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,2,128,1,float16,fp8,0,2.191978613535563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,2,128,1,fp8,fp8,0,1.5920693079630535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,2,128,1,float16,float16,0,2.245445410410563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,4,128,1,float16,float16,0,2.2086559931437173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,4,128,1,float16,fp8,0,2.1829706827799478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,4,128,1,fp8,fp8,0,1.5962239901224773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,8,128,1,float16,float16,0,2.218074639638265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,8,128,1,fp8,fp8,0,1.599295934041341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,40,8,128,1,float16,fp8,0,2.19870932896932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,1,128,1,float16,fp8,0,22.198521931966145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,1,128,1,fp8,fp8,0,15.314404805501303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,1,128,1,float16,float16,0,23.1887690226237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,2,128,1,fp8,fp8,0,15.354208628336588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,2,128,1,float16,float16,0,22.64759572347005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,2,128,1,float16,fp8,0,22.475440979003906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,4,128,1,float16,float16,0,22.71375020345052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,4,128,1,fp8,fp8,0,15.275530497233072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,4,128,1,float16,fp8,0,22.374916076660156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,8,128,1,float16,float16,0,23.160293579101562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,8,128,1,float16,fp8,0,22.907808939615887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,40,128,1,float16,float16,0,11.497114817301432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,40,8,128,1,fp8,fp8,0,15.398773193359375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,40,128,1,float16,fp8,0,11.565205891927084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,1,128,1,float16,float16,0,11.276368459065756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,1,128,1,fp8,fp8,0,7.701407750447591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,1,128,1,float16,fp8,0,11.433226267496744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,2,128,1,float16,float16,0,11.368863423665365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,2,128,1,float16,fp8,0,11.389045715332031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,2,128,1,fp8,fp8,0,7.716784159342448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,4,128,1,float16,float16,0,11.273648579915365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,40,128,1,fp8,fp8,0,8.02508290608724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,4,128,1,float16,fp8,0,11.363653818766275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,4,128,1,fp8,fp8,0,7.8827253977457685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,8,128,1,float16,float16,0,11.464618682861328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,8,128,1,float16,fp8,0,11.388261159261068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,40,8,128,1,fp8,fp8,0,7.776325225830078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,1,128,1,float16,float16,0,5.690170923868815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,40,128,1,float16,float16,0,5.6359202067057295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,1,128,1,fp8,fp8,0,3.832869211832682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,1,128,1,float16,fp8,0,5.312293370564778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,40,128,1,float16,fp8,0,5.702314376831055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,2,128,1,float16,float16,0,5.610149383544922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,2,128,1,fp8,fp8,0,3.8465598424275718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,2,128,1,float16,fp8,0,5.724485397338867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,40,128,1,fp8,fp8,0,3.981093406677246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,4,128,1,float16,float16,0,5.375205357869466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,4,128,1,float16,fp8,0,5.516031901041667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,4,128,1,fp8,fp8,0,3.8533172607421875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,8,128,1,float16,float16,0,5.520544052124023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,8,128,1,float16,fp8,0,5.4558455149332685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,1,128,1,float16,float16,0,2.7774025599161782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,40,8,128,1,fp8,fp8,0,3.907525380452474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,40,128,1,float16,float16,0,2.82640012105306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,1,128,1,float16,fp8,0,2.757941246032715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,40,128,1,float16,fp8,0,2.779930750528971
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,40,128,1,fp8,fp8,0,2.082325299580892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,1,128,1,fp8,fp8,0,2.0116213162740073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,2,128,1,float16,float16,0,2.7849225997924805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,2,128,1,fp8,fp8,0,2.0115200678507485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,2,128,1,float16,fp8,0,2.8014294306437173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,4,128,1,float16,float16,0,2.7993014653523765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,4,128,1,float16,fp8,0,2.7551679611206055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,4,128,1,fp8,fp8,0,2.0180160204569497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,8,128,1,float16,float16,0,2.8000427881876626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,8,128,1,float16,fp8,0,2.7580318450927734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,40,8,128,1,fp8,fp8,0,2.0238827069600425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,40,128,1,float16,float16,0,1.5280853907267253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,1,128,1,float16,float16,0,1.504042625427246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,40,128,1,float16,fp8,0,1.4943413734436035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,40,128,1,fp8,fp8,0,1.1349813143412273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,1,128,1,float16,fp8,0,1.4830133120218914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,1,128,1,fp8,fp8,0,1.1073119640350342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,2,128,1,float16,float16,0,1.5034240086873372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,2,128,1,float16,fp8,0,1.5047732988993328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,2,128,1,fp8,fp8,0,1.106613318125407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,4,128,1,float16,float16,0,1.5087839762369792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,4,128,1,float16,fp8,0,1.4856053988138835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,4,128,1,fp8,fp8,0,1.1112533410390217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,8,128,1,float16,float16,0,1.5056586265563965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,8,128,1,float16,fp8,0,1.4921813011169434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,40,8,128,1,fp8,fp8,0,1.1149760087331135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,1,128,1,fp8,fp8,0,9.241578420003256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,1,128,1,float16,float16,0,13.421984354654947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,2,128,1,float16,float16,0,13.293818155924479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,2,128,1,float16,fp8,0,13.241503397623697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,2,128,1,fp8,fp8,0,9.262128194173178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,1,128,1,float16,fp8,0,13.321210225423178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,4,128,1,float16,float16,0,13.741808573404947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,4,128,1,fp8,fp8,0,9.277695973714193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,4,128,1,float16,fp8,0,13.283098856608072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,8,128,1,fp8,fp8,0,9.437615712483725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,8,128,1,float16,fp8,0,13.336746215820312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,40,8,128,1,float16,float16,0,13.382677714029947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,40,128,1,float16,float16,0,6.849962870279948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,1,128,1,float16,float16,0,6.592282613118489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,1,128,1,float16,fp8,0,6.665615717569987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,1,128,1,fp8,fp8,0,4.609285354614258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,40,128,1,float16,fp8,0,6.846975962320964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,2,128,1,float16,float16,0,6.560239791870117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,2,128,1,float16,fp8,0,6.515546798706055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,2,128,1,fp8,fp8,0,4.61788272857666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,40,128,1,fp8,fp8,0,4.828336079915364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,4,128,1,float16,float16,0,6.539445241292317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,4,128,1,float16,fp8,0,6.626688003540039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,4,128,1,fp8,fp8,0,4.7496693929036455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,8,128,1,float16,float16,0,6.654058456420898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,8,128,1,float16,fp8,0,6.391829172770183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,40,8,128,1,fp8,fp8,0,4.663802782694499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,1,128,1,float16,float16,0,3.3070240020751953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,1,128,1,float16,fp8,0,3.175914764404297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,1,128,1,fp8,fp8,0,2.361626625061035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,2,128,1,float16,float16,0,3.215087890625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,40,128,1,float16,fp8,0,3.3344427744547525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,40,128,1,float16,float16,0,3.309909184773763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,40,128,1,fp8,fp8,0,2.4885546366373696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,2,128,1,fp8,fp8,0,2.367856025695801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,2,128,1,float16,fp8,0,3.3595412572224936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,4,128,1,float16,float16,0,3.2365172704060874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,4,128,1,fp8,fp8,0,2.377002716064453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,4,128,1,float16,fp8,0,3.2392587661743164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,8,128,1,float16,float16,0,3.301605224609375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,8,128,1,float16,fp8,0,3.2307891845703125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,40,8,128,1,fp8,fp8,0,2.3937759399414062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,40,128,1,float16,fp8,0,1.7054452896118164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,40,128,1,float16,float16,0,1.7443946202596028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,1,128,1,float16,float16,0,1.6874507268269856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,1,128,1,float16,fp8,0,1.66265074412028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,1,128,1,fp8,fp8,0,1.2554453214009602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,2,128,1,float16,float16,0,1.6951252619425456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,2,128,1,float16,fp8,0,1.669376055399577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,40,128,1,fp8,fp8,0,1.3130559921264648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,4,128,1,float16,float16,0,1.693413257598877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,2,128,1,fp8,fp8,0,1.25709335009257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,4,128,1,float16,fp8,0,1.6707359949747722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,4,128,1,fp8,fp8,0,1.261349360148112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,8,128,1,float16,float16,0,1.703978697458903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,8,128,1,float16,fp8,0,1.6795573234558105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,40,8,128,1,fp8,fp8,0,1.2692800362904866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,40,128,1,float16,float16,0,0.9346293608347574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,1,128,1,float16,float16,0,0.9309279918670654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,40,128,1,float16,fp8,0,0.9364960193634033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,1,128,1,float16,fp8,0,0.9166239897410074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,40,128,1,fp8,fp8,0,0.7316319942474365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,1,128,1,fp8,fp8,0,0.7069066365559896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,2,128,1,fp8,fp8,0,0.7070293426513672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,2,128,1,float16,fp8,0,0.9190719922383627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,2,128,1,float16,float16,0,0.9326612949371338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,4,128,1,float16,fp8,0,0.922106663386027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,4,128,1,float16,float16,0,0.9415093262990316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,4,128,1,fp8,fp8,0,0.7096640268961588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,8,128,1,float16,float16,0,0.9372853438059489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,8,128,1,float16,fp8,0,0.9207359949747721
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,40,8,128,1,fp8,fp8,0,0.7141973177591959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,1,128,1,float16,fp8,0,12.834864298502604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,1,128,1,fp8,fp8,0,9.413962682088217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,2,128,1,fp8,fp8,0,9.342149098714193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,2,128,1,float16,fp8,0,12.922261555989584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,1,128,1,float16,float16,0,13.135477701822916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,2,128,1,float16,float16,0,13.027109781901041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,4,128,1,float16,float16,0,13.00719960530599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,4,128,1,float16,fp8,0,12.802244822184244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,4,128,1,fp8,fp8,0,9.657957077026367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,8,128,1,fp8,fp8,0,9.458799997965494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,8,128,1,float16,float16,0,13.124783833821615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,40,128,1,float16,float16,0,6.604490915934245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,40,8,128,1,float16,fp8,0,13.054704030354818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,40,128,1,float16,fp8,0,6.6957441965738935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,1,128,1,float16,float16,0,6.572160085042317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,1,128,1,fp8,fp8,0,4.698410669962565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,1,128,1,float16,fp8,0,6.413733164469401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,2,128,1,float16,float16,0,6.263893127441406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,2,128,1,float16,fp8,0,6.4050343831380205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,2,128,1,fp8,fp8,0,4.709514617919922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,4,128,1,float16,float16,0,6.541418711344401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,4,128,1,float16,fp8,0,6.340431849161784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,40,128,1,fp8,fp8,0,5.030751864115397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,4,128,1,fp8,fp8,0,4.726890563964844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,8,128,1,float16,float16,0,6.54038937886556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,8,128,1,float16,fp8,0,6.421690622965495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,1,128,1,float16,float16,0,3.1155519485473633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,40,8,128,1,fp8,fp8,0,4.769290606180827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,1,128,1,float16,fp8,0,3.0603628158569336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,1,128,1,fp8,fp8,0,2.376250743865967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,2,128,1,float16,float16,0,3.1416266759236655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,40,128,1,float16,float16,0,3.3589439392089844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,40,128,1,float16,fp8,0,3.212042808532715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,40,128,1,fp8,fp8,0,2.5503039360046387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,2,128,1,float16,fp8,0,3.093722661336263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,2,128,1,fp8,fp8,0,2.385749340057373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,4,128,1,float16,fp8,0,3.0721120834350586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,4,128,1,float16,float16,0,3.140000025431315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,4,128,1,fp8,fp8,0,2.39355198542277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,8,128,1,float16,float16,0,3.139589309692383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,8,128,1,float16,fp8,0,3.09390926361084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,40,8,128,1,fp8,fp8,0,2.4144320487976074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,1,128,1,float16,float16,0,1.6301493644714355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,40,128,1,float16,float16,0,1.6825067202250164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,1,128,1,fp8,fp8,0,1.2401920159657795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,1,128,1,float16,fp8,0,1.5821119944254558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,2,128,1,float16,float16,0,1.6152639389038086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,40,128,1,float16,fp8,0,1.6519145965576172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,40,128,1,fp8,fp8,0,1.3334666887919109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,2,128,1,float16,fp8,0,1.6050079663594563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,2,128,1,fp8,fp8,0,1.2452159722646077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,4,128,1,float16,float16,0,1.6242027282714844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,4,128,1,float16,fp8,0,1.5901494026184082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,4,128,1,fp8,fp8,0,1.2485653559366863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,8,128,1,float16,float16,0,1.6261547406514485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,8,128,1,float16,fp8,0,1.6022027333577473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,40,8,128,1,fp8,fp8,0,1.2582026322682698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,1,128,1,float16,float16,0,0.8662292957305908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,40,128,1,float16,fp8,0,0.8729546864827474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,40,128,1,fp8,fp8,0,0.7177973588307699
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,40,128,1,float16,float16,0,0.8875946998596191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,1,128,1,float16,fp8,0,0.8506293296813965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,1,128,1,fp8,fp8,0,0.6717920303344727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,2,128,1,float16,float16,0,0.8734986782073975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,2,128,1,float16,fp8,0,0.8503466447194418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,2,128,1,fp8,fp8,0,0.6761333147684733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,4,128,1,float16,float16,0,0.870405356089274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,4,128,1,fp8,fp8,0,0.6772533257802328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,8,128,1,float16,float16,0,0.8735840320587158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,4,128,1,float16,fp8,0,0.8535040219624838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,8,128,1,float16,fp8,0,0.8636319637298584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,40,8,128,1,fp8,fp8,0,0.6837813059488932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,40,128,1,float16,float16,0,0.499722679456075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,1,128,1,float16,float16,0,0.49422399202982586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,40,128,1,float16,fp8,0,0.492304007212321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,1,128,1,fp8,fp8,0,0.37195734182993573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,40,128,1,fp8,fp8,0,0.3956640164057414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,1,128,1,float16,fp8,0,0.485973318417867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,2,128,1,float16,float16,0,0.4957173268000285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,2,128,1,fp8,fp8,0,0.3727039893468221
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,2,128,1,float16,fp8,0,0.48655998706817627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,4,128,1,float16,float16,0,0.4970719814300537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,4,128,1,float16,fp8,0,0.4880160093307495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,4,128,1,fp8,fp8,0,0.3730986515680949
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,8,128,1,float16,float16,0,0.4989973306655884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,8,128,1,float16,fp8,0,0.4889226754506429
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,40,8,128,1,fp8,fp8,0,0.3763573169708252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,1,128,1,float16,float16,0,7.923242568969727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,1,128,1,float16,fp8,0,7.724629084269206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,2,128,1,float16,float16,0,7.917695999145508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,1,128,1,fp8,fp8,0,5.99833615620931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,2,128,1,fp8,fp8,0,5.978677113850911
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,2,128,1,float16,fp8,0,7.877317428588867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,4,128,1,float16,float16,0,7.950762430826823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,4,128,1,float16,fp8,0,7.6764265696207685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,4,128,1,fp8,fp8,0,6.0059254964192705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,8,128,1,fp8,fp8,0,6.063893636067708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,8,128,1,float16,float16,0,8.104880015055338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,40,8,128,1,float16,fp8,0,7.945056279500325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,40,128,1,float16,fp8,0,4.018255869547526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,1,128,1,float16,float16,0,3.8068691889444985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,40,128,1,float16,float16,0,4.114789326985677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,1,128,1,fp8,fp8,0,3.000368118286133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,1,128,1,float16,fp8,0,3.7575200398763022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,2,128,1,float16,float16,0,3.869130770365397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,2,128,1,float16,fp8,0,3.740629196166992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,2,128,1,fp8,fp8,0,3.0093065897623696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,40,128,1,fp8,fp8,0,3.252613385518392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,4,128,1,fp8,fp8,0,3.0238879521687827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,4,128,1,float16,fp8,0,3.8453493118286133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,8,128,1,float16,float16,0,3.942922592163086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,4,128,1,float16,float16,0,3.834458669026693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,8,128,1,float16,fp8,0,3.8618666330973306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,1,128,1,float16,float16,0,1.9551893870035808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,40,8,128,1,fp8,fp8,0,3.055813471476237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,1,128,1,float16,fp8,0,1.894613265991211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,1,128,1,fp8,fp8,0,1.5268747011820476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,40,128,1,float16,fp8,0,2.025322596232096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,40,128,1,float16,float16,0,2.06276798248291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,40,128,1,fp8,fp8,0,1.6669546763102214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,2,128,1,float16,float16,0,1.9590613047281902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,2,128,1,float16,fp8,0,1.8963200251261394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,2,128,1,fp8,fp8,0,1.529706637064616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,4,128,1,float16,float16,0,1.9486079216003418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,4,128,1,float16,fp8,0,1.9052693049112956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,4,128,1,fp8,fp8,0,1.5414667129516602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,8,128,1,float16,fp8,0,1.9427040417989094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,8,128,1,fp8,fp8,0,1.556447982788086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,40,8,128,1,float16,float16,0,1.9566720326741536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,1,128,1,float16,float16,0,1.0138719876607258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,40,128,1,float16,float16,0,1.067903995513916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,1,128,1,float16,fp8,0,0.9986453056335449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,1,128,1,fp8,fp8,0,0.806389331817627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,2,128,1,float16,float16,0,1.0156426429748535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,2,128,1,float16,fp8,0,0.9965919653574625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,40,128,1,float16,fp8,0,1.0495200157165527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,40,128,1,fp8,fp8,0,0.8811573187510172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,2,128,1,fp8,fp8,0,0.8092586994171143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,4,128,1,float16,fp8,0,0.9987733364105225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,4,128,1,float16,float16,0,1.0263466835021973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,4,128,1,fp8,fp8,0,0.8125759760538737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,8,128,1,float16,fp8,0,1.0053013165791829
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,8,128,1,float16,float16,0,1.0281439622243245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,40,8,128,1,fp8,fp8,0,0.8219040234883627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,1,128,1,float16,float16,0,0.5542240142822266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,1,128,1,float16,fp8,0,0.5417066812515259
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,2,128,1,float16,float16,0,0.5558666785558065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,1,128,1,fp8,fp8,0,0.4468586842219035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,40,128,1,float16,float16,0,0.5722293456395467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,40,128,1,float16,fp8,0,0.5618346532185873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,40,128,1,fp8,fp8,0,0.480565349260966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,2,128,1,float16,fp8,0,0.5432693163553873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,2,128,1,fp8,fp8,0,0.4487413167953491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,4,128,1,float16,float16,0,0.5566346645355225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,4,128,1,float16,fp8,0,0.54585067431132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,8,128,1,float16,fp8,0,0.5496746699015299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,8,128,1,fp8,fp8,0,0.4532959858576457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,8,128,1,float16,float16,0,0.5607466697692871
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,40,4,128,1,fp8,fp8,0,0.4493866761525472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,1,128,1,float16,float16,0,0.32555200656255084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,40,128,1,float16,float16,0,0.33263999223709106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,40,128,1,float16,fp8,0,0.325872004032135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,1,128,1,float16,fp8,0,0.31703466176986694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,1,128,1,fp8,fp8,0,0.25124800205230713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,40,128,1,fp8,fp8,0,0.27138133843739826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,2,128,1,float16,fp8,0,0.3187146584192912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,2,128,1,float16,float16,0,0.3264639973640442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,2,128,1,fp8,fp8,0,0.2520959973335266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,4,128,1,float16,float16,0,0.327455997467041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,4,128,1,float16,fp8,0,0.32045867045720416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,4,128,1,fp8,fp8,0,0.2523519992828369
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,8,128,1,float16,float16,0,0.3293439944585164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,8,128,1,float16,fp8,0,0.3230293393135071
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,40,8,128,1,fp8,fp8,0,0.255237340927124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,1,128,1,float16,float16,0,8.240255991617838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,1,128,1,float16,fp8,0,7.869642893473308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,1,128,1,fp8,fp8,0,6.65988286336263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,2,128,1,float16,fp8,0,7.930490493774414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,2,128,1,fp8,fp8,0,6.67241096496582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,2,128,1,float16,float16,0,8.284629185994467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,4,128,1,float16,float16,0,8.28384017944336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,4,128,1,float16,fp8,0,8.036895751953125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,4,128,1,fp8,fp8,0,6.714192072550456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,8,128,1,float16,float16,0,8.503738403320312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,8,128,1,float16,fp8,0,8.054453531901041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,40,8,128,1,fp8,fp8,0,6.767178853352864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,40,128,1,float16,float16,0,4.384991963704427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,1,128,1,float16,float16,0,4.034000078837077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,1,128,1,float16,fp8,0,3.8919200897216797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,1,128,1,fp8,fp8,0,3.293386777242025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,2,128,1,float16,float16,0,4.099125226338704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,2,128,1,fp8,fp8,0,3.2988319396972656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,2,128,1,float16,fp8,0,3.8729546864827475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,40,128,1,float16,fp8,0,4.29423459370931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,4,128,1,float16,float16,0,4.05128542582194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,40,128,1,fp8,fp8,0,3.627408027648926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,4,128,1,float16,fp8,0,3.9149866104125977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,4,128,1,fp8,fp8,0,3.318613370259603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,8,128,1,float16,float16,0,4.100261370340983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,1,128,1,float16,float16,0,2.003653367360433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,8,128,1,float16,fp8,0,3.9512106577555337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,40,8,128,1,fp8,fp8,0,3.3553921381632485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,1,128,1,float16,fp8,0,1.93612273534139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,40,128,1,float16,float16,0,2.19484806060791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,40,128,1,float16,fp8,0,2.149360020955404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,1,128,1,fp8,fp8,0,1.647856076558431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,40,128,1,fp8,fp8,0,1.8393759727478027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,2,128,1,float16,float16,0,1.9948053359985352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,2,128,1,float16,fp8,0,1.942794640858968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,2,128,1,fp8,fp8,0,1.6545599301656086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,4,128,1,float16,float16,0,2.0056373278299966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,4,128,1,fp8,fp8,0,1.6656053860982258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,4,128,1,float16,fp8,0,1.9536852836608887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,8,128,1,float16,float16,0,2.0214239756266275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,8,128,1,float16,fp8,0,1.9708693822224934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,1,128,1,float16,float16,0,1.0301439762115479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,40,8,128,1,fp8,fp8,0,1.6870932579040527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,1,128,1,fp8,fp8,0,0.8549226919809977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,1,128,1,float16,fp8,0,1.0043093363444011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,40,128,1,float16,float16,0,1.1199999650319417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,2,128,1,float16,float16,0,1.0324160257975261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,40,128,1,float16,fp8,0,1.093178669611613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,40,128,1,fp8,fp8,0,0.9534506797790527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,2,128,1,float16,fp8,0,1.0060160160064697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,2,128,1,fp8,fp8,0,0.8564960161844889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,4,128,1,float16,float16,0,1.0383093357086182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,4,128,1,float16,fp8,0,1.0092213153839111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,4,128,1,fp8,fp8,0,0.8626666863759359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,8,128,1,float16,fp8,0,1.0231413046518962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,8,128,1,float16,float16,0,1.0434026718139648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,40,128,1,float16,float16,0,0.5854986508687338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,40,8,128,1,fp8,fp8,0,0.8747626940409342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,1,128,1,float16,float16,0,0.5498346487681071
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,40,128,1,float16,fp8,0,0.575818657875061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,1,128,1,fp8,fp8,0,0.4585813283920288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,1,128,1,float16,fp8,0,0.5365920066833496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,40,128,1,fp8,fp8,0,0.5104426542917887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,2,128,1,float16,float16,0,0.5504853328069051
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,4,128,1,float16,float16,0,0.5556426843007406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,2,128,1,fp8,fp8,0,0.46217068036397296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,4,128,1,fp8,fp8,0,0.46450666586558026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,4,128,1,float16,fp8,0,0.5393813451131185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,2,128,1,float16,fp8,0,0.5372053384780884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,8,128,1,float16,float16,0,0.5562826792399088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,1,128,1,float16,fp8,0,0.30105600754419964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,8,128,1,float16,fp8,0,0.5435359875361124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,40,8,128,1,fp8,fp8,0,0.4715199867884318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,40,128,1,float16,float16,0,0.32390934228897095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,40,128,1,float16,fp8,0,0.31850665807724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,40,128,1,fp8,fp8,0,0.28097599744796753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,1,128,1,float16,float16,0,0.3096959988276164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,1,128,1,fp8,fp8,0,0.2513226668039958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,2,128,1,float16,float16,0,0.31061333417892456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,2,128,1,float16,fp8,0,0.3034506638844808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,2,128,1,fp8,fp8,0,0.25092800458272296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,4,128,1,float16,fp8,0,0.3036053379376729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,4,128,1,float16,float16,0,0.3121119936307271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,4,128,1,fp8,fp8,0,0.2534666657447815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,8,128,1,float16,float16,0,0.3139626582463582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,8,128,1,float16,fp8,0,0.3057973384857178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,40,8,128,1,fp8,fp8,0,0.25646400451660156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,40,128,1,float16,float16,0,0.1848586599032084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,40,128,1,float16,fp8,0,0.18150933583577475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,40,128,1,fp8,fp8,0,0.16502933700879416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,1,128,1,float16,float16,0,0.17244799931844076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,1,128,1,float16,fp8,0,0.1678719917933146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,1,128,1,fp8,fp8,0,0.15081600348154703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,2,128,1,float16,float16,0,0.17361599206924438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,2,128,1,float16,fp8,0,0.1690453290939331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,2,128,1,fp8,fp8,0,0.1516693333784739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,4,128,1,float16,float16,0,0.17416000366210938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,4,128,1,fp8,fp8,0,0.15288533767064413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,8,128,1,float16,float16,0,0.17450666427612305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,8,128,1,float16,fp8,0,0.17094399531682333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,8,128,1,fp8,fp8,0,0.1543359955151876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,40,4,128,1,float16,fp8,0,0.1691360076268514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,1,128,1,float16,float16,0,5.28221861521403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,1,128,1,float16,fp8,0,5.12277348836263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,1,128,1,fp8,fp8,0,4.4648745854695635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,2,128,1,fp8,fp8,0,4.470778783162435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,2,128,1,float16,float16,0,5.298421223958333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,2,128,1,float16,fp8,0,5.05841064453125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,4,128,1,float16,float16,0,5.340778350830078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,4,128,1,float16,fp8,0,5.085018793741862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,4,128,1,fp8,fp8,0,4.489247957865397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,8,128,1,fp8,fp8,0,4.516640027364095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,8,128,1,float16,float16,0,5.414255777994792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,40,8,128,1,float16,fp8,0,5.190762519836426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,40,128,1,float16,float16,0,2.9013118743896484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,1,128,1,float16,float16,0,2.60750404993693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,40,128,1,float16,fp8,0,2.793365478515625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,1,128,1,float16,fp8,0,2.484517256418864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,1,128,1,fp8,fp8,0,2.204106648763021
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,2,128,1,float16,float16,0,2.6036853790283203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,2,128,1,float16,fp8,0,2.494511922200521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,2,128,1,fp8,fp8,0,2.210053284962972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,4,128,1,float16,float16,0,2.6279145876566568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,40,128,1,fp8,fp8,0,2.4749706586201987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,4,128,1,fp8,fp8,0,2.2281920115152993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,4,128,1,float16,fp8,0,2.504330635070801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,8,128,1,float16,float16,0,2.653157393137614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,8,128,1,float16,fp8,0,2.5396533012390137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,1,128,1,float16,float16,0,1.296074628829956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,40,8,128,1,fp8,fp8,0,2.256336053212484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,1,128,1,float16,fp8,0,1.2578773498535156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,1,128,1,fp8,fp8,0,1.1128640174865723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,40,128,1,float16,float16,0,1.4561386108398438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,40,128,1,float16,fp8,0,1.412511984507243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,2,128,1,float16,float16,0,1.3003839651743572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,2,128,1,float16,fp8,0,1.264346679051717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,40,128,1,fp8,fp8,0,1.263919989267985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,2,128,1,fp8,fp8,0,1.1168373425801594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,4,128,1,float16,float16,0,1.3094293276468914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,4,128,1,float16,fp8,0,1.2717866897583008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,4,128,1,fp8,fp8,0,1.1232213179270427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,8,128,1,float16,float16,0,1.3209280172983806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,8,128,1,float16,fp8,0,1.2869919935862224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,1,128,1,float16,float16,0,0.6767306327819824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,40,8,128,1,fp8,fp8,0,1.1415306727091472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,40,128,1,float16,float16,0,0.7494826316833496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,40,128,1,float16,fp8,0,0.7281066576639811
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,1,128,1,float16,fp8,0,0.6570080121358236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,1,128,1,fp8,fp8,0,0.585098663965861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,40,128,1,fp8,fp8,0,0.6588533322016398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,2,128,1,float16,float16,0,0.6802026430765787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,2,128,1,float16,fp8,0,0.6598399877548218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,2,128,1,fp8,fp8,0,0.5850826501846313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,4,128,1,float16,float16,0,0.6823946634928385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,4,128,1,float16,fp8,0,0.6626933415730795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,4,128,1,fp8,fp8,0,0.5906986792882284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,1,128,1,float16,float16,0,0.36689599355061847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,8,128,1,float16,fp8,0,0.6705866654713949
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,8,128,1,float16,float16,0,0.6884693304697672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,40,8,128,1,fp8,fp8,0,0.5976853370666504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,40,128,1,float16,float16,0,0.3983999888102214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,40,128,1,fp8,fp8,0,0.35684800148010254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,40,128,1,float16,fp8,0,0.38704001903533936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,1,128,1,float16,fp8,0,0.3569973309834798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,1,128,1,fp8,fp8,0,0.3176959951718648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,2,128,1,fp8,fp8,0,0.31963199377059937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,2,128,1,float16,fp8,0,0.35831467310587567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,2,128,1,float16,float16,0,0.36939199765523273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,4,128,1,float16,float16,0,0.3699733416239421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,4,128,1,float16,fp8,0,0.3601813316345215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,4,128,1,fp8,fp8,0,0.3226826588312785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,8,128,1,float16,float16,0,0.37301866213480633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,8,128,1,float16,fp8,0,0.36397333939870197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,40,8,128,1,fp8,fp8,0,0.3269439935684204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,1,128,1,float16,float16,0,0.20982933044433594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,40,128,1,float16,float16,0,0.2244159976641337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,40,128,1,float16,fp8,0,0.219050665696462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,40,128,1,fp8,fp8,0,0.1983893314997355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,1,128,1,float16,fp8,0,0.2028800050417582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,1,128,1,fp8,fp8,0,0.17679999272028604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,2,128,1,float16,float16,0,0.2100480000178019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,2,128,1,float16,fp8,0,0.20423465967178345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,4,128,1,float16,float16,0,0.21202667554219565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,4,128,1,float16,fp8,0,0.20556267102559408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,8,128,1,float16,float16,0,0.2144213318824768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,8,128,1,float16,fp8,0,0.20827200015385947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,2,128,1,fp8,fp8,0,0.17828800280888876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,4,128,1,fp8,fp8,0,0.17800533771514893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,40,8,128,1,fp8,fp8,0,0.18101867039998373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,1,128,1,float16,float16,0,0.11982933680216472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,40,128,1,float16,fp8,0,0.12854400277137756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,40,128,1,float16,float16,0,0.13024000326792398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,40,128,1,fp8,fp8,0,0.12119999527931213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,1,128,1,float16,fp8,0,0.11781332890192668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,2,128,1,float16,float16,0,0.12071466445922852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,1,128,1,fp8,fp8,0,0.10806933045387268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,2,128,1,float16,fp8,0,0.11748266220092773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,4,128,1,float16,fp8,0,0.11852799852689107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,2,128,1,fp8,fp8,0,0.10917866230010986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,4,128,1,float16,float16,0,0.12106666962305705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,8,128,1,float16,float16,0,0.12225066622098286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,8,128,1,fp8,fp8,0,0.11406933267911275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,4,128,1,fp8,fp8,0,0.11170666416486104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,40,8,128,1,float16,fp8,0,0.11927466591199239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,1,128,1,float16,fp8,0,4.845226605733235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,1,128,1,float16,float16,0,4.969290733337402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,1,128,1,fp8,fp8,0,4.330293337504069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,2,128,1,float16,fp8,0,4.981077194213867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,2,128,1,float16,float16,0,5.101685206095378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,2,128,1,fp8,fp8,0,4.386357307434082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,4,128,1,float16,float16,0,5.204853375752767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,4,128,1,float16,fp8,0,5.11568546295166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,4,128,1,fp8,fp8,0,4.43070920308431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,8,128,1,float16,fp8,0,5.265791893005371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,8,128,1,float16,float16,0,5.259375890096028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,40,8,128,1,fp8,fp8,0,4.5263519287109375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,40,128,1,float16,float16,0,3.206005414326986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,1,128,1,float16,float16,0,2.4376373291015625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,40,128,1,float16,fp8,0,3.070144017537435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,1,128,1,float16,fp8,0,2.4317386945088706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,1,128,1,fp8,fp8,0,2.1614452997843423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,2,128,1,float16,float16,0,2.452357292175293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,2,128,1,float16,fp8,0,2.447007973988851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,2,128,1,fp8,fp8,0,2.1669227282206216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,4,128,1,float16,float16,0,2.479050636291504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,4,128,1,float16,fp8,0,2.5411465962727866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,4,128,1,fp8,fp8,0,2.2188852628072104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,40,128,1,fp8,fp8,0,2.6619359652201333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,8,128,1,float16,float16,0,2.618922710418701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,1,128,1,float16,float16,0,1.2349279721577961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,8,128,1,fp8,fp8,0,2.261781374613444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,40,8,128,1,float16,fp8,0,2.607349395751953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,1,128,1,float16,fp8,0,1.233845313390096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,40,128,1,float16,float16,0,1.6115519205729167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,1,128,1,fp8,fp8,0,1.0547573566436768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,40,128,1,float16,fp8,0,1.5405066808064778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,2,128,1,float16,float16,0,1.2414720058441162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,40,128,1,fp8,fp8,0,1.3390080134073894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,2,128,1,float16,fp8,0,1.2386879920959473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,2,128,1,fp8,fp8,0,1.090453306833903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,4,128,1,float16,float16,0,1.2549813588460286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,4,128,1,float16,fp8,0,1.2544960180918376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,4,128,1,fp8,fp8,0,1.1138506730397542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,8,128,1,float16,float16,0,1.2965226968129475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,8,128,1,float16,fp8,0,1.2776319980621338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,40,128,1,float16,float16,0,0.8158346811930338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,40,8,128,1,fp8,fp8,0,1.1341439882914226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,1,128,1,float16,float16,0,0.6311519940694174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,1,128,1,float16,fp8,0,0.6324106852213541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,1,128,1,fp8,fp8,0,0.5422186851501465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,40,128,1,float16,fp8,0,0.7710080146789551
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,40,128,1,fp8,fp8,0,0.6783520380655924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,2,128,1,float16,float16,0,0.6348533233006796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,2,128,1,fp8,fp8,0,0.5469013452529907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,2,128,1,float16,fp8,0,0.6346559921900431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,4,128,1,float16,float16,0,0.6439520120620728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,4,128,1,float16,fp8,0,0.6432373523712158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,4,128,1,fp8,fp8,0,0.5552586714426676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,8,128,1,float16,float16,0,0.6582293510437012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,8,128,1,float16,fp8,0,0.6542293230692545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,40,8,128,1,fp8,fp8,0,0.574890653292338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,40,128,1,float16,float16,0,0.4178933302561442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,1,128,1,float16,float16,0,0.3279306689898173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,40,128,1,float16,fp8,0,0.4006506601969401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,1,128,1,float16,fp8,0,0.327455997467041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,1,128,1,fp8,fp8,0,0.2835093339284261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,40,128,1,fp8,fp8,0,0.3466453154881795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,2,128,1,float16,float16,0,0.3306186596552531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,2,128,1,fp8,fp8,0,0.28574933608373004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,2,128,1,float16,fp8,0,0.33153067032496136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,4,128,1,float16,float16,0,0.33605865637461346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,4,128,1,float16,fp8,0,0.33560534318288165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,4,128,1,fp8,fp8,0,0.28908799091974896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,8,128,1,float16,float16,0,0.3418666521708171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,8,128,1,float16,fp8,0,0.3400426705678304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,1,128,1,float16,float16,0,0.1783733367919922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,40,8,128,1,fp8,fp8,0,0.29447466135025024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,40,128,1,float16,float16,0,0.22315200169881186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,40,128,1,float16,fp8,0,0.21599467595418295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,1,128,1,float16,fp8,0,0.17828800280888876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,40,128,1,fp8,fp8,0,0.18634132544199625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,1,128,1,fp8,fp8,0,0.1490613321463267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,2,128,1,float16,float16,0,0.1803200046221415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,2,128,1,float16,fp8,0,0.18030933539072672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,4,128,1,float16,float16,0,0.1837973395983378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,4,128,1,float16,fp8,0,0.18389866749445596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,4,128,1,fp8,fp8,0,0.15266666809717813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,8,128,1,float16,float16,0,0.1853546698888143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,2,128,1,fp8,fp8,0,0.15094400445620218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,8,128,1,float16,fp8,0,0.18607999881108603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,40,8,128,1,fp8,fp8,0,0.1546239952246348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,40,128,1,float16,float16,0,0.12803199887275696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,1,128,1,float16,float16,0,0.09842133522033691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,40,128,1,float16,fp8,0,0.12433066964149475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,40,128,1,fp8,fp8,0,0.10136000315348308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,1,128,1,fp8,fp8,0,0.08431999882062276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,2,128,1,float16,float16,0,0.09828799962997437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,1,128,1,float16,fp8,0,0.09826667110125224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,2,128,1,float16,fp8,0,0.09844266374905904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,2,128,1,fp8,fp8,0,0.08517866333325703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,4,128,1,float16,fp8,0,0.09928533434867859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,4,128,1,float16,float16,0,0.09915199875831604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,4,128,1,fp8,fp8,0,0.08690133690834045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,40,128,1,float16,float16,0,0.06788266698519389
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,8,128,1,fp8,fp8,0,0.08808533350626628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,40,128,1,float16,fp8,0,0.06577600042025249
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,8,128,1,float16,float16,0,0.10019200046857198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,40,128,1,fp8,fp8,0,0.06044800082842509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,40,8,128,1,float16,fp8,0,0.10055466492970784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,1,128,1,float16,float16,0,0.05644266804059347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,1,128,1,float16,fp8,0,0.0566240002711614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,1,128,1,fp8,fp8,0,0.04921066761016846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,2,128,1,float16,float16,0,0.05644266804059347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,2,128,1,fp8,fp8,0,0.049695998430252075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,4,128,1,fp8,fp8,0,0.05274133384227753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,4,128,1,float16,fp8,0,0.057904000083605446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,2,128,1,float16,fp8,0,0.05683733522891998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,4,128,1,float16,float16,0,0.05685866872469584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,8,128,1,float16,fp8,0,0.05955733358860016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,8,128,1,fp8,fp8,0,0.053360000252723694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,40,8,128,1,float16,float16,0,0.058687999844551086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,1,128,1,float16,float16,0,3.768085479736328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,1,128,1,float16,fp8,0,3.776437441507975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,2,128,1,float16,float16,0,3.802234649658203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,2,128,1,fp8,fp8,0,3.604895909627279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,2,128,1,float16,fp8,0,3.8883307774861655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,4,128,1,float16,float16,0,4.0425920486450195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,1,128,1,fp8,fp8,0,3.576650619506836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,4,128,1,float16,fp8,0,4.014128049214681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,4,128,1,fp8,fp8,0,3.6965761184692383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,8,128,1,float16,fp8,0,4.173221270243327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,8,128,1,float16,float16,0,4.2082773844401045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,40,8,128,1,fp8,fp8,0,3.7845280965169272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,1,128,1,float16,float16,0,1.897557258605957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,1,128,1,float16,fp8,0,1.8971786499023438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,40,128,1,float16,float16,0,2.6927626927693686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,2,128,1,float16,float16,0,1.9149707158406575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,40,128,1,float16,fp8,0,2.553317387898763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,1,128,1,fp8,fp8,0,1.7726453145345051
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,2,128,1,float16,fp8,0,1.923866589864095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,2,128,1,fp8,fp8,0,1.7990026473999023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,40,128,1,fp8,fp8,0,2.292949358622233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,4,128,1,float16,float16,0,1.9441653887430828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,4,128,1,fp8,fp8,0,1.8407680193583171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,4,128,1,float16,fp8,0,2.00654935836792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,8,128,1,float16,float16,0,2.0803732872009277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,1,128,1,float16,float16,0,0.9637920061747233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,8,128,1,float16,fp8,0,2.0722667376200357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,40,8,128,1,fp8,fp8,0,1.885514736175537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,1,128,1,float16,fp8,0,0.9648533662160238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,1,128,1,fp8,fp8,0,0.8671733538309733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,40,128,1,float16,float16,0,1.346826712290446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,40,128,1,float16,fp8,0,1.2830506960550945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,40,128,1,fp8,fp8,0,1.155626694361369
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,2,128,1,float16,float16,0,0.9717546304066976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,2,128,1,float16,fp8,0,0.9696640173594157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,2,128,1,fp8,fp8,0,0.9013280073801676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,4,128,1,float16,float16,0,0.985642671585083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,4,128,1,float16,fp8,0,0.9820586840311686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,4,128,1,fp8,fp8,0,0.9225653012593588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,8,128,1,float16,float16,0,1.0188053448994954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,1,128,1,float16,float16,0,0.4941493272781372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,8,128,1,float16,fp8,0,1.0185653368632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,40,8,128,1,fp8,fp8,0,0.9423413276672363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,1,128,1,float16,fp8,0,0.49278398354848224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,40,128,1,float16,float16,0,0.6827893257141113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,1,128,1,fp8,fp8,0,0.44550931453704834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,40,128,1,float16,fp8,0,0.6390719811121622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,40,128,1,fp8,fp8,0,0.5820959806442261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,2,128,1,float16,float16,0,0.49937065442403156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,2,128,1,float16,fp8,0,0.49726398785909015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,2,128,1,fp8,fp8,0,0.4494560162226359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,4,128,1,float16,float16,0,0.5061866839726766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,4,128,1,float16,fp8,0,0.5034879843393961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,4,128,1,fp8,fp8,0,0.457477331161499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,8,128,1,float16,float16,0,0.5201706488927206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,8,128,1,float16,fp8,0,0.5176159938176473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,40,8,128,1,fp8,fp8,0,0.4758346478144328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,1,128,1,float16,float16,0,0.2558613419532776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,1,128,1,float16,fp8,0,0.2555946707725525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,1,128,1,fp8,fp8,0,0.23417067527770996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,2,128,1,float16,float16,0,0.2585546573003133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,2,128,1,float16,fp8,0,0.2584000031153361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,40,128,1,fp8,fp8,0,0.29765333731969196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,40,128,1,float16,float16,0,0.34973867734273273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,40,128,1,float16,fp8,0,0.33161600430806476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,2,128,1,fp8,fp8,0,0.23538132508595785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,4,128,1,float16,float16,0,0.26239999135335285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,4,128,1,fp8,fp8,0,0.240064005057017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,4,128,1,float16,fp8,0,0.2630506753921509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,8,128,1,float16,float16,0,0.2702133258183797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,8,128,1,fp8,fp8,0,0.24473067124684653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,40,128,1,float16,float16,0,0.18653867642084757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,40,8,128,1,float16,fp8,0,0.26874667406082153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,1,128,1,float16,float16,0,0.13901866475741068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,1,128,1,float16,fp8,0,0.13872533043225607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,2,128,1,float16,float16,0,0.14019200205802917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,40,128,1,fp8,fp8,0,0.16191466649373373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,1,128,1,fp8,fp8,0,0.12917332847913107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,40,128,1,float16,fp8,0,0.17849600315093994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,2,128,1,fp8,fp8,0,0.13004266222318014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,2,128,1,float16,fp8,0,0.1402293344338735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,4,128,1,float16,float16,0,0.14411200086275736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,4,128,1,float16,fp8,0,0.14453333616256714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,4,128,1,fp8,fp8,0,0.13191999991734824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,8,128,1,float16,float16,0,0.14644799629847208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,8,128,1,fp8,fp8,0,0.13501866658528647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,40,128,1,float16,float16,0,0.11534399787584941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,40,8,128,1,float16,fp8,0,0.14577600359916687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,40,128,1,float16,fp8,0,0.11181867122650146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,1,128,1,float16,float16,0,0.08142399787902832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,40,128,1,fp8,fp8,0,0.08918933073679607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,1,128,1,fp8,fp8,0,0.07328533132870992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,1,128,1,float16,fp8,0,0.08125333487987518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,2,128,1,float16,float16,0,0.08151466647783916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,2,128,1,fp8,fp8,0,0.07435200115044911
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,4,128,1,float16,fp8,0,0.08236266672611237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,4,128,1,float16,float16,0,0.08269333342711131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,4,128,1,fp8,fp8,0,0.07541866600513458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,2,128,1,float16,fp8,0,0.08140799899895985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,8,128,1,float16,fp8,0,0.08333866794904073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,8,128,1,fp8,fp8,0,0.07747733096281688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,40,8,128,1,float16,float16,0,0.08382933338483174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,40,128,1,float16,fp8,0,0.05698133508364359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,40,128,1,float16,float16,0,0.05922133227189382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,40,128,1,fp8,fp8,0,0.05384533107280731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,1,128,1,float16,fp8,0,0.04785599807898203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,1,128,1,float16,float16,0,0.04785599807898203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,1,128,1,fp8,fp8,0,0.04252799848715464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,2,128,1,float16,fp8,0,0.04855999847253164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,2,128,1,fp8,fp8,0,0.043365334471066795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,4,128,1,float16,float16,0,0.04797866443792979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,2,128,1,float16,float16,0,0.04850133260091146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,4,128,1,float16,fp8,0,0.049285332361857094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,8,128,1,float16,float16,0,0.04888000090916952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,8,128,1,float16,fp8,0,0.04910400013128916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,8,128,1,fp8,fp8,0,0.0469706654548645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,40,128,1,float16,fp8,0,0.03522666543722153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,40,128,1,float16,float16,0,0.03563733398914337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,40,4,128,1,fp8,fp8,0,0.0447573314110438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,40,128,1,fp8,fp8,0,0.03274133304754893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,1,128,1,float16,float16,0,0.03152533372243246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,1,128,1,float16,fp8,0,0.03150933235883713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,1,128,1,fp8,fp8,0,0.03002133220434189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,2,128,1,float16,float16,0,0.0317493329445521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,2,128,1,float16,fp8,0,0.031845333675543465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,4,128,1,float16,float16,0,0.03233599911133448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,4,128,1,float16,fp8,0,0.03252800057331721
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,4,128,1,fp8,fp8,0,0.03088533381621043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,8,128,1,float16,float16,0,0.03206400076548258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,2,128,1,fp8,fp8,0,0.030192000170548756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,8,128,1,float16,fp8,0,0.032960000137488045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,40,8,128,1,fp8,fp8,0,0.031871999303499855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,1,128,1,float16,float16,0,1.6366027196248372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,1,128,1,fp8,fp8,0,1.5094134012858074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,2,128,1,float16,float16,0,1.6505920092264812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,1,128,1,float16,fp8,0,1.6364852587382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,2,128,1,float16,fp8,0,1.6465919812520344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,4,128,1,float16,float16,0,1.6771732966105144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,2,128,1,fp8,fp8,0,1.5570346514383953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,4,128,1,float16,fp8,0,1.699461301167806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,4,128,1,fp8,fp8,0,1.6100212732950847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,8,128,1,float16,fp8,0,1.7971946398417156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,8,128,1,float16,float16,0,1.8155520757039387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,40,8,128,1,fp8,fp8,0,1.659989356994629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,40,128,1,float16,float16,0,1.2391786575317383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,40,128,1,float16,fp8,0,1.172597328821818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,1,128,1,float16,float16,0,0.8314133485158285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,1,128,1,fp8,fp8,0,0.749621311823527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,1,128,1,float16,fp8,0,0.8315733273824056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,2,128,1,float16,float16,0,0.8386987050374349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,2,128,1,float16,fp8,0,0.8364693323771158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,4,128,1,float16,float16,0,0.8530560334523519
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,2,128,1,fp8,fp8,0,0.7831733226776123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,40,128,1,fp8,fp8,0,1.0466079711914062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,4,128,1,float16,fp8,0,0.850757360458374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,4,128,1,fp8,fp8,0,0.8060213724772135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,8,128,1,float16,float16,0,0.8917866547902426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,1,128,1,float16,float16,0,0.42607466379801434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,8,128,1,fp8,fp8,0,0.83023468653361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,40,128,1,float16,float16,0,0.6236000061035156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,1,128,1,float16,fp8,0,0.4261653423309326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,40,8,128,1,float16,fp8,0,0.8863680362701416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,40,128,1,float16,fp8,0,0.5811839898427328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,1,128,1,fp8,fp8,0,0.38896532853444415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,40,128,1,fp8,fp8,0,0.527567982673645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,2,128,1,float16,float16,0,0.4310186703999837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,2,128,1,float16,fp8,0,0.43067200978597003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,2,128,1,fp8,fp8,0,0.3938346703847249
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,4,128,1,float16,float16,0,0.4383840163548787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,4,128,1,float16,fp8,0,0.43616533279418945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,4,128,1,fp8,fp8,0,0.40326400597890216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,8,128,1,float16,float16,0,0.45289067427317303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,8,128,1,float16,fp8,0,0.44946666558583576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,1,128,1,float16,float16,0,0.2206666668256124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,1,128,1,float16,fp8,0,0.22012799978256226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,40,128,1,float16,float16,0,0.32093334197998047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,40,8,128,1,fp8,fp8,0,0.42060800393422443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,1,128,1,fp8,fp8,0,0.20524799823760986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,40,128,1,fp8,fp8,0,0.27128533522288006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,2,128,1,float16,float16,0,0.22239466508229574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,40,128,1,float16,fp8,0,0.3038240075111389
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,2,128,1,float16,fp8,0,0.22404799858729044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,2,128,1,fp8,fp8,0,0.2074133356412252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,4,128,1,float16,float16,0,0.22759999831517538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,4,128,1,float16,fp8,0,0.22800000508626303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,4,128,1,fp8,fp8,0,0.2109760046005249
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,8,128,1,float16,float16,0,0.234607994556427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,8,128,1,fp8,fp8,0,0.21675199270248413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,40,8,128,1,float16,fp8,0,0.23319466908772787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,40,128,1,float16,float16,0,0.17262399196624756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,1,128,1,float16,float16,0,0.12078400452931722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,40,128,1,float16,fp8,0,0.16425599654515585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,1,128,1,float16,fp8,0,0.12169599533081055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,40,128,1,fp8,fp8,0,0.14935466647148132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,1,128,1,fp8,fp8,0,0.11565333604812622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,2,128,1,float16,fp8,0,0.12275733550389607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,2,128,1,fp8,fp8,0,0.1163146694501241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,2,128,1,float16,float16,0,0.12283733487129211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,4,128,1,float16,fp8,0,0.12594133615493774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,4,128,1,fp8,fp8,0,0.11879466970761617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,4,128,1,float16,float16,0,0.12519466876983643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,8,128,1,float16,float16,0,0.12749333182970682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,8,128,1,float16,fp8,0,0.12756266196568808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,40,8,128,1,fp8,fp8,0,0.12036266922950745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,40,128,1,float16,float16,0,0.10893332958221436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,40,128,1,float16,fp8,0,0.10390933354695638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,1,128,1,float16,fp8,0,0.07217066486676534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,40,128,1,fp8,fp8,0,0.08490133285522461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,1,128,1,float16,float16,0,0.07197333375612895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,2,128,1,float16,float16,0,0.07221333185831706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,1,128,1,fp8,fp8,0,0.06553066770235698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,2,128,1,fp8,fp8,0,0.06691733499368031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,4,128,1,float16,float16,0,0.0726506660381953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,4,128,1,float16,fp8,0,0.0734506646792094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,2,128,1,float16,fp8,0,0.07268266876538594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,8,128,1,float16,float16,0,0.07546666761239369
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,8,128,1,float16,fp8,0,0.07506133119265239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,8,128,1,fp8,fp8,0,0.07016533116499583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,40,4,128,1,fp8,fp8,0,0.06814399858315785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,40,128,1,float16,float16,0,0.05565333366394043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,40,128,1,float16,fp8,0,0.05429333448410034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,40,128,1,fp8,fp8,0,0.051962668697039284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,1,128,1,float16,float16,0,0.043935999274253845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,1,128,1,float16,fp8,0,0.04357333481311798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,1,128,1,fp8,fp8,0,0.04035199930270513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,2,128,1,fp8,fp8,0,0.040661332507928215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,2,128,1,float16,fp8,0,0.044495999813079834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,4,128,1,float16,float16,0,0.04481600224971771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,2,128,1,float16,float16,0,0.04456000030040741
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,4,128,1,float16,fp8,0,0.04465066889921824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,4,128,1,fp8,fp8,0,0.04260266820589701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,8,128,1,float16,float16,0,0.04506133496761322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,8,128,1,float16,fp8,0,0.04573333263397217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,40,8,128,1,fp8,fp8,0,0.04461333155632019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,40,128,1,float16,float16,0,0.03422933320204417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,40,128,1,fp8,fp8,0,0.03251733382542928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,1,128,1,float16,float16,0,0.030640001098314922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,40,128,1,float16,fp8,0,0.03418133407831192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,1,128,1,float16,fp8,0,0.030847998956839245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,2,128,1,float16,float16,0,0.031066666046778362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,1,128,1,fp8,fp8,0,0.02957333376010259
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,2,128,1,float16,fp8,0,0.030847998956839245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,2,128,1,fp8,fp8,0,0.029909332593282063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,4,128,1,float16,float16,0,0.031178665657838184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,4,128,1,fp8,fp8,0,0.030773334205150604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,8,128,1,float16,float16,0,0.03213333338499069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,8,128,1,float16,fp8,0,0.03182400017976761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,8,128,1,fp8,fp8,0,0.03201599915822347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,40,4,128,1,float16,fp8,0,0.032314665615558624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,40,128,1,float16,float16,0,0.025487999121348064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,1,128,1,float16,float16,0,0.02359466751416524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,40,128,1,fp8,fp8,0,0.023669332265853882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,40,128,1,float16,fp8,0,0.025226667523384094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,1,128,1,float16,fp8,0,0.02359466751416524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,1,128,1,fp8,fp8,0,0.022143999735514324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,2,128,1,float16,fp8,0,0.02409599969784419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,2,128,1,fp8,fp8,0,0.022458667556444805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,4,128,1,float16,float16,0,0.023936000963052113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,4,128,1,fp8,fp8,0,0.0229120006163915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,2,128,1,float16,float16,0,0.02385599911212921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,4,128,1,float16,fp8,0,0.02399466683467229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,8,128,1,float16,float16,0,0.024031999210516613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,8,128,1,float16,fp8,0,0.02418133368094762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,40,8,128,1,fp8,fp8,0,0.023061332603295643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,1,128,1,float16,float16,0,0.7744373480478922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,1,128,1,float16,fp8,0,0.774943987528483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,1,128,1,fp8,fp8,0,0.7502826849619547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,2,128,1,float16,float16,0,0.7798826694488525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,2,128,1,float16,fp8,0,0.7801012992858887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,2,128,1,fp8,fp8,0,0.7800959746042887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,4,128,1,float16,float16,0,0.7932319641113281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,4,128,1,float16,fp8,0,0.7929440339406332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,4,128,1,fp8,fp8,0,0.8103040059407552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,8,128,1,float16,float16,0,0.8220160007476807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,8,128,1,float16,fp8,0,0.8204692999521891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,40,8,128,1,fp8,fp8,0,0.8297812938690186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,40,128,1,float16,float16,0,0.6438773473103842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,40,128,1,float16,fp8,0,0.6083626747131348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,1,128,1,float16,float16,0,0.4002666473388672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,1,128,1,fp8,fp8,0,0.3871306578318278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,2,128,1,float16,fp8,0,0.4025813341140747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,1,128,1,float16,fp8,0,0.39985068639119464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,2,128,1,float16,float16,0,0.4044959942499797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,2,128,1,fp8,fp8,0,0.3915146589279175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,4,128,1,float16,float16,0,0.41126398245493573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,40,128,1,fp8,fp8,0,0.528223991394043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,4,128,1,float16,fp8,0,0.4081813494364421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,4,128,1,fp8,fp8,0,0.40113067626953125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,8,128,1,float16,fp8,0,0.4235413471857707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,8,128,1,fp8,fp8,0,0.4193813403447469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,40,8,128,1,float16,float16,0,0.4280213514963786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,40,128,1,float16,float16,0,0.33532798290252686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,1,128,1,float16,float16,0,0.20844799280166626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,1,128,1,float16,fp8,0,0.20875734090805054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,1,128,1,fp8,fp8,0,0.20588266849517822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,40,128,1,float16,fp8,0,0.3160319924354553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,40,128,1,fp8,fp8,0,0.26872533559799194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,2,128,1,float16,fp8,0,0.210533340771993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,2,128,1,float16,float16,0,0.21107733249664307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,2,128,1,fp8,fp8,0,0.20732800165812174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,4,128,1,float16,float16,0,0.21500800053278604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,4,128,1,float16,fp8,0,0.2161173423131307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,4,128,1,fp8,fp8,0,0.21314666668574014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,8,128,1,float16,float16,0,0.22378132740656534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,8,128,1,float16,fp8,0,0.22299200296401978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,40,8,128,1,fp8,fp8,0,0.21798932552337646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,40,128,1,float16,float16,0,0.1779306729634603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,1,128,1,float16,float16,0,0.11574932932853699
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,40,128,1,fp8,fp8,0,0.1455946664015452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,1,128,1,float16,fp8,0,0.11635200182596843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,1,128,1,fp8,fp8,0,0.1149173378944397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,40,128,1,float16,fp8,0,0.170250674088796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,2,128,1,float16,float16,0,0.11687466502189636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,2,128,1,float16,fp8,0,0.11769599715868632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,2,128,1,fp8,fp8,0,0.11629866560300191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,4,128,1,float16,float16,0,0.12097066640853882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,4,128,1,float16,fp8,0,0.12006400028864543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,4,128,1,fp8,fp8,0,0.11735467116038005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,8,128,1,float16,float16,0,0.12299733360608418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,8,128,1,float16,fp8,0,0.12304533521334331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,40,8,128,1,fp8,fp8,0,0.11987732847531636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,40,128,1,float16,float16,0,0.10274666547775269
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,1,128,1,float16,float16,0,0.0686773310105006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,40,128,1,float16,fp8,0,0.09894399841626485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,1,128,1,float16,fp8,0,0.06871466835339864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,40,128,1,fp8,fp8,0,0.08438400427500407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,1,128,1,fp8,fp8,0,0.06611733138561249
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,2,128,1,float16,float16,0,0.06879466772079468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,2,128,1,float16,fp8,0,0.0694400022427241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,2,128,1,fp8,fp8,0,0.06613866488138835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,4,128,1,float16,float16,0,0.0699893335501353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,4,128,1,fp8,fp8,0,0.06773333251476288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,8,128,1,float16,fp8,0,0.07187733550866444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,8,128,1,float16,float16,0,0.07270933190981548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,4,128,1,float16,fp8,0,0.07027199864387512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,40,8,128,1,fp8,fp8,0,0.07054399947325389
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,40,128,1,float16,float16,0,0.05406400064627329
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,40,128,1,float16,fp8,0,0.05220800141493479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,40,128,1,fp8,fp8,0,0.05099200208981832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,1,128,1,float16,float16,0,0.042223999897638954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,1,128,1,float16,fp8,0,0.04221333563327789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,2,128,1,float16,float16,0,0.04186666508515676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,1,128,1,fp8,fp8,0,0.04035199930270513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,2,128,1,float16,fp8,0,0.04285866518815359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,2,128,1,fp8,fp8,0,0.04106666644414266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,4,128,1,float16,float16,0,0.04279999931653341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,4,128,1,float16,fp8,0,0.043136000633239746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,4,128,1,fp8,fp8,0,0.042303999265034996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,8,128,1,float16,float16,0,0.043882668018341064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,8,128,1,fp8,fp8,0,0.04463466505209605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,40,8,128,1,float16,fp8,0,0.04354133208592733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,40,128,1,float16,float16,0,0.033717334270477295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,40,128,1,float16,fp8,0,0.03299733251333237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,40,128,1,fp8,fp8,0,0.03302400062481562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,1,128,1,float16,float16,0,0.031231999397277832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,1,128,1,float16,fp8,0,0.030928000807762146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,1,128,1,fp8,fp8,0,0.02943466603755951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,2,128,1,float16,fp8,0,0.03136533250411352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,2,128,1,float16,float16,0,0.031210665901501972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,2,128,1,fp8,fp8,0,0.029904000461101532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,4,128,1,float16,float16,0,0.03151999910672506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,4,128,1,float16,fp8,0,0.032042667269706726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,4,128,1,fp8,fp8,0,0.030506665507952373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,8,128,1,float16,float16,0,0.032560000816980995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,8,128,1,float16,fp8,0,0.03267733256022135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,40,8,128,1,fp8,fp8,0,0.0317493329445521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,40,128,1,float16,fp8,0,0.024698667228221893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,40,128,1,float16,float16,0,0.023887999355793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,1,128,1,float16,float16,0,0.02274133265018463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,40,128,1,fp8,fp8,0,0.024090667565663654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,1,128,1,float16,fp8,0,0.02274666726589203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,1,128,1,fp8,fp8,0,0.022458667556444805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,2,128,1,float16,fp8,0,0.023221333821614582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,2,128,1,float16,float16,0,0.02258133391539256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,4,128,1,float16,float16,0,0.022874665757020313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,4,128,1,float16,fp8,0,0.02327999969323476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,2,128,1,fp8,fp8,0,0.02219199885924657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,4,128,1,fp8,fp8,0,0.023050665855407715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,8,128,1,float16,float16,0,0.02312533309062322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,8,128,1,float16,fp8,0,0.023455999791622162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,40,128,1,float16,float16,0,0.02042666698495547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,40,128,1,float16,fp8,0,0.021146667500336964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,40,8,128,1,fp8,fp8,0,0.02292266736427943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,40,128,1,fp8,fp8,0,0.020549333343903225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,1,128,1,float16,float16,0,0.020047999918460846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,1,128,1,float16,fp8,0,0.020101333657900494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,1,128,1,fp8,fp8,0,0.01958400011062622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,2,128,1,float16,fp8,0,0.020746666938066483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,2,128,1,float16,float16,0,0.020154666155576706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,4,128,1,float16,fp8,0,0.02054399996995926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,4,128,1,float16,float16,0,0.019930666933457058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,2,128,1,fp8,fp8,0,0.019823999454577763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,4,128,1,fp8,fp8,0,0.019973333925008774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,8,128,1,float16,float16,0,0.019839999576409657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,8,128,1,float16,fp8,0,0.020608000457286835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,40,8,128,1,fp8,fp8,0,0.020389333367347717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,1,128,1,float16,float16,0,0.49750932057698566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,1,128,1,float16,fp8,0,0.4969866673151652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,1,128,1,fp8,fp8,0,0.5188746849695841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,2,128,1,float16,float16,0,0.5016586780548096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,2,128,1,float16,fp8,0,0.4996266762415568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,4,128,1,float16,float16,0,0.5079040129979452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,2,128,1,fp8,fp8,0,0.5261439879735311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,4,128,1,float16,fp8,0,0.5062079826990763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,4,128,1,fp8,fp8,0,0.5374399820963541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,8,128,1,float16,fp8,0,0.5192639827728271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,8,128,1,fp8,fp8,0,0.5538880030314127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,40,8,128,1,float16,float16,0,0.5219253301620483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,40,128,1,float16,float16,0,0.33537065982818604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,1,128,1,float16,float16,0,0.2520853281021118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,40,128,1,float16,fp8,0,0.31650133927663165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,1,128,1,float16,fp8,0,0.2521333297093709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,1,128,1,fp8,fp8,0,0.2701440056165059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,2,128,1,float16,fp8,0,0.25436800718307495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,40,128,1,fp8,fp8,0,0.33368531862894696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,2,128,1,float16,float16,0,0.25521065791447956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,2,128,1,fp8,fp8,0,0.2723093430201213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,4,128,1,float16,float16,0,0.2595093250274658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,4,128,1,fp8,fp8,0,0.2774239977200826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,4,128,1,float16,fp8,0,0.2595679958661397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,8,128,1,float16,fp8,0,0.2648106614748637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,8,128,1,float16,float16,0,0.26665065685908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,40,128,1,float16,fp8,0,0.1692426602045695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,40,128,1,float16,float16,0,0.17800533771514893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,40,8,128,1,fp8,fp8,0,0.28276799122492474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,1,128,1,float16,float16,0,0.13580800096193948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,1,128,1,float16,fp8,0,0.1364959975083669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,1,128,1,fp8,fp8,0,0.14763200283050537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,40,128,1,fp8,fp8,0,0.17723200718561807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,2,128,1,float16,float16,0,0.13700800140698752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,2,128,1,float16,fp8,0,0.13724266489346823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,4,128,1,float16,float16,0,0.13959999879201254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,2,128,1,fp8,fp8,0,0.1483840048313141
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,4,128,1,float16,fp8,0,0.1401653289794922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,4,128,1,fp8,fp8,0,0.15033599734306335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,8,128,1,float16,float16,0,0.14272532860438028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,8,128,1,float16,fp8,0,0.14179199934005737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,40,8,128,1,fp8,fp8,0,0.1523360013961792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,1,128,1,float16,float16,0,0.0768746683994929
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,40,128,1,float16,float16,0,0.09840533137321472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,1,128,1,float16,fp8,0,0.07671999931335449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,40,128,1,fp8,fp8,0,0.09961066643397014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,2,128,1,float16,float16,0,0.07708799839019775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,40,128,1,float16,fp8,0,0.09488532940546672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,1,128,1,fp8,fp8,0,0.0832533339659373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,2,128,1,float16,fp8,0,0.07734400033950806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,2,128,1,fp8,fp8,0,0.08335999647776286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,4,128,1,float16,float16,0,0.07797333101431529
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,4,128,1,float16,fp8,0,0.07881066699822743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,4,128,1,fp8,fp8,0,0.08507200082143147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,8,128,1,fp8,fp8,0,0.08716799815495808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,8,128,1,float16,fp8,0,0.08058133224646251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,40,8,128,1,float16,float16,0,0.08014933268229167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,40,128,1,float16,float16,0,0.05324266850948334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,1,128,1,float16,float16,0,0.044826666514078774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,40,128,1,float16,fp8,0,0.05078400174776713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,40,128,1,fp8,fp8,0,0.05782933533191681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,1,128,1,float16,fp8,0,0.044735997915267944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,2,128,1,float16,fp8,0,0.04553066690762838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,1,128,1,fp8,fp8,0,0.04744000236193339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,2,128,1,fp8,fp8,0,0.047824000318845115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,4,128,1,float16,float16,0,0.04577599962552389
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,4,128,1,float16,fp8,0,0.04666133224964142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,2,128,1,float16,float16,0,0.044778664906819664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,4,128,1,fp8,fp8,0,0.04966933528582255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,8,128,1,float16,float16,0,0.04678933322429657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,8,128,1,fp8,fp8,0,0.05102399984995524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,40,128,1,float16,float16,0,0.03328000009059906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,40,128,1,float16,fp8,0,0.033402666449546814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,40,8,128,1,float16,fp8,0,0.04719999929269155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,1,128,1,float16,float16,0,0.03192000091075897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,1,128,1,fp8,fp8,0,0.032933334509531655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,1,128,1,float16,fp8,0,0.03230399886767069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,40,128,1,fp8,fp8,0,0.036271999279658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,2,128,1,float16,fp8,0,0.031898667414983116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,2,128,1,float16,float16,0,0.03175999969244003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,4,128,1,float16,float16,0,0.032560000816980995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,4,128,1,float16,fp8,0,0.03244800120592117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,8,128,1,float16,fp8,0,0.03293866664171219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,4,128,1,fp8,fp8,0,0.03383466601371765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,2,128,1,fp8,fp8,0,0.03295466552178065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,8,128,1,fp8,fp8,0,0.03500800083080927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,40,8,128,1,float16,float16,0,0.03242133309443792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,40,128,1,fp8,fp8,0,0.02722666660944621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,40,128,1,float16,fp8,0,0.02603200078010559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,40,128,1,float16,float16,0,0.026208000878492992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,1,128,1,float16,float16,0,0.025568000972270966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,1,128,1,fp8,fp8,0,0.025605333348115284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,2,128,1,float16,fp8,0,0.025439999997615814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,2,128,1,fp8,fp8,0,0.026015999416510265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,4,128,1,float16,float16,0,0.02533866713444392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,4,128,1,float16,fp8,0,0.025722667574882507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,2,128,1,float16,float16,0,0.025285333395004272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,1,128,1,float16,fp8,0,0.025402667621771496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,4,128,1,fp8,fp8,0,0.026880001028378803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,8,128,1,float16,float16,0,0.025706666211287182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,8,128,1,float16,fp8,0,0.02605333427588145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,40,8,128,1,fp8,fp8,0,0.026501332720120747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,40,128,1,float16,float16,0,0.019365333020687103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,1,128,1,float16,fp8,0,0.01940800001223882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,1,128,1,float16,float16,0,0.018837332725524902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,1,128,1,fp8,fp8,0,0.0194560003777345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,40,128,1,float16,fp8,0,0.019834666202465694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,40,128,1,fp8,fp8,0,0.020703999946514767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,2,128,1,float16,float16,0,0.019066666563351948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,2,128,1,float16,fp8,0,0.018944000204404194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,2,128,1,fp8,fp8,0,0.020015999674797058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,4,128,1,float16,fp8,0,0.01926933353145917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,4,128,1,float16,float16,0,0.01871466636657715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,8,128,1,float16,fp8,0,0.019519999623298645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,40,128,1,float16,fp8,0,0.0186666672428449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,8,128,1,fp8,fp8,0,0.0201706662774086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,40,128,1,float16,float16,0,0.018640000373125076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,4,128,1,fp8,fp8,0,0.020047999918460846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,40,128,1,fp8,fp8,0,0.01932799940307935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,40,8,128,1,float16,float16,0,0.019280000279347103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,1,128,1,float16,fp8,0,0.018245333184798557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,1,128,1,float16,float16,0,0.018538666268189747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,2,128,1,float16,float16,0,0.01848000039656957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,1,128,1,fp8,fp8,0,0.01904533306757609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,2,128,1,fp8,fp8,0,0.01889066646496455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,2,128,1,float16,fp8,0,0.018581333259741466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,8,128,1,float16,float16,0,0.018186666071414948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,4,128,1,float16,fp8,0,0.01826133330663045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,4,128,1,fp8,fp8,0,0.019066666563351948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,4,128,1,float16,float16,0,0.0183999997874101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,8,128,1,float16,fp8,0,0.018725333114465077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,40,8,128,1,fp8,fp8,0,0.019023999571800232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,1,128,1,float16,fp8,0,0.33312533299128216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,2,128,1,float16,float16,0,0.338864008585612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,2,128,1,fp8,fp8,0,0.4201066493988037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,1,128,1,fp8,fp8,0,0.4174240032831828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,1,128,1,float16,float16,0,0.3327466646830241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,4,128,1,float16,float16,0,0.34889598687489826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,2,128,1,float16,fp8,0,0.33960533142089844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,4,128,1,float16,fp8,0,0.348527987798055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,4,128,1,fp8,fp8,0,0.4272853136062622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,8,128,1,float16,float16,0,0.35816534360249835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,8,128,1,float16,fp8,0,0.35555732250213623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,40,128,1,float16,float16,0,0.21676266193389893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,40,128,1,float16,fp8,0,0.2085226575533549
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,40,8,128,1,fp8,fp8,0,0.43299734592437744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,1,128,1,float16,float16,0,0.1742453376452128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,1,128,1,float16,fp8,0,0.17485332489013672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,1,128,1,fp8,fp8,0,0.22156266371409097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,2,128,1,float16,float16,0,0.17713600397109985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,2,128,1,float16,fp8,0,0.17791465918223062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,40,128,1,fp8,fp8,0,0.25194666783014935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,2,128,1,fp8,fp8,0,0.22272533178329468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,4,128,1,float16,float16,0,0.18268267313639322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,4,128,1,float16,fp8,0,0.1822026570638021
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,8,128,1,float16,float16,0,0.1869493325551351
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,4,128,1,fp8,fp8,0,0.22362132867177328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,8,128,1,fp8,fp8,0,0.22630399465560913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,40,8,128,1,float16,fp8,0,0.18551466862360635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,40,128,1,float16,float16,0,0.11662399768829346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,1,128,1,float16,float16,0,0.09397866328557332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,1,128,1,float16,fp8,0,0.09455999732017517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,40,128,1,float16,fp8,0,0.1137600044409434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,40,128,1,fp8,fp8,0,0.13763200243314108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,1,128,1,fp8,fp8,0,0.12166933218638103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,2,128,1,float16,float16,0,0.09434133768081665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,2,128,1,fp8,fp8,0,0.12216533223787944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,4,128,1,float16,float16,0,0.0965226689974467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,4,128,1,float16,fp8,0,0.09654933214187622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,4,128,1,fp8,fp8,0,0.12308800220489502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,8,128,1,float16,float16,0,0.10107200344403584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,2,128,1,float16,fp8,0,0.09505599737167358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,8,128,1,float16,fp8,0,0.10000532865524292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,40,8,128,1,fp8,fp8,0,0.12522133191426596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,1,128,1,float16,float16,0,0.05406933526198069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,40,128,1,float16,float16,0,0.06316799918810527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,1,128,1,fp8,fp8,0,0.06779199838638306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,40,128,1,float16,fp8,0,0.06018666426340739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,40,128,1,fp8,fp8,0,0.07679466903209686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,1,128,1,float16,fp8,0,0.05341866612434387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,2,128,1,float16,float16,0,0.05310399830341339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,2,128,1,fp8,fp8,0,0.06717866659164429
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,2,128,1,float16,fp8,0,0.053871999184290566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,4,128,1,float16,float16,0,0.05429866909980774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,4,128,1,float16,fp8,0,0.0547680010398229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,4,128,1,fp8,fp8,0,0.06945600112279256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,8,128,1,float16,float16,0,0.05434666574001312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,8,128,1,float16,fp8,0,0.05585066477457682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,40,8,128,1,fp8,fp8,0,0.07066133121649425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,1,128,1,float16,float16,0,0.0347626656293869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,40,128,1,float16,float16,0,0.03613866617282232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,40,128,1,float16,fp8,0,0.03618133316437403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,40,128,1,fp8,fp8,0,0.04393066465854645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,1,128,1,fp8,fp8,0,0.04114133367935816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,2,128,1,float16,float16,0,0.0347626656293869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,2,128,1,float16,fp8,0,0.03474666674931844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,2,128,1,fp8,fp8,0,0.04101333270470301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,4,128,1,float16,float16,0,0.03526400029659271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,4,128,1,float16,fp8,0,0.034789333740870156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,1,128,1,float16,fp8,0,0.034815999368826546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,4,128,1,fp8,fp8,0,0.041989331444104515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,8,128,1,float16,float16,0,0.03523733218510946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,8,128,1,fp8,fp8,0,0.043562665581703186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,40,128,1,float16,float16,0,0.026965332527955372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,40,128,1,float16,fp8,0,0.02651199946800868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,40,8,128,1,float16,fp8,0,0.03600533306598663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,1,128,1,float16,float16,0,0.02626666675011317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,40,128,1,fp8,fp8,0,0.031109333038330078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,1,128,1,float16,fp8,0,0.02630399912595749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,1,128,1,fp8,fp8,0,0.03025600065787633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,2,128,1,float16,float16,0,0.026719999810059864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,2,128,1,float16,fp8,0,0.02624000112215678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,2,128,1,fp8,fp8,0,0.030640001098314922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,4,128,1,float16,float16,0,0.026378666361172993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,4,128,1,fp8,fp8,0,0.030991998811562855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,8,128,1,float16,fp8,0,0.02693866689999898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,8,128,1,fp8,fp8,0,0.03148799886306127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,4,128,1,float16,fp8,0,0.026133333643277485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,40,8,128,1,float16,float16,0,0.026687999566396076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,40,128,1,float16,float16,0,0.02142400046189626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,1,128,1,float16,float16,0,0.020981334149837494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,40,128,1,fp8,fp8,0,0.024853333830833435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,40,128,1,float16,fp8,0,0.02160000056028366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,1,128,1,float16,fp8,0,0.021503999829292297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,2,128,1,fp8,fp8,0,0.023792001108328503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,4,128,1,float16,float16,0,0.021685334543387096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,4,128,1,float16,fp8,0,0.02088533341884613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,2,128,1,float16,fp8,0,0.021562665700912476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,1,128,1,fp8,fp8,0,0.0232640008131663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,4,128,1,fp8,fp8,0,0.023936000963052113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,2,128,1,float16,float16,0,0.021125334004561108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,8,128,1,float16,float16,0,0.021087999145189922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,8,128,1,float16,fp8,0,0.02142400046189626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,40,8,128,1,fp8,fp8,0,0.023658665517965954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,40,128,1,fp8,fp8,0,0.02025066688656807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,40,128,1,float16,float16,0,0.01781333362062772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,1,128,1,float16,float16,0,0.0179626668492953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,1,128,1,float16,fp8,0,0.018277333428462345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,1,128,1,fp8,fp8,0,0.01894933357834816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,40,128,1,float16,fp8,0,0.01826133330663045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,2,128,1,float16,float16,0,0.01749333366751671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,2,128,1,float16,fp8,0,0.01829333355029424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,2,128,1,fp8,fp8,0,0.01912533367673556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,4,128,1,float16,float16,0,0.018031999468803406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,4,128,1,fp8,fp8,0,0.01904533306757609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,4,128,1,float16,fp8,0,0.01809599995613098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,8,128,1,float16,fp8,0,0.017952000101407368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,8,128,1,fp8,fp8,0,0.019167999426523846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,40,8,128,1,float16,float16,0,0.019434666881958645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,40,128,1,float16,fp8,0,0.017770666629076004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,1,128,1,float16,float16,0,0.017466666797796886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,40,128,1,float16,float16,0,0.01664000004529953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,40,128,1,fp8,fp8,0,0.018629333625237148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,1,128,1,float16,fp8,0,0.01758933315674464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,1,128,1,fp8,fp8,0,0.01830400029818217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,2,128,1,fp8,fp8,0,0.01922133316596349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,2,128,1,float16,fp8,0,0.01775466650724411
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,4,128,1,fp8,fp8,0,0.01870399961868922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,4,128,1,float16,fp8,0,0.01736533393462499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,2,128,1,float16,float16,0,0.017258666455745697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,4,128,1,float16,float16,0,0.017045332739750545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,8,128,1,float16,float16,0,0.01708799973130226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,8,128,1,float16,fp8,0,0.017829333742459614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,40,8,128,1,fp8,fp8,0,0.018677332748969395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,1,128,1,float16,float16,0,0.2797653277715047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,1,128,1,float16,fp8,0,0.2802613377571106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,2,128,1,float16,float16,0,0.2826773325602214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,2,128,1,float16,fp8,0,0.2832640012105306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,1,128,1,fp8,fp8,0,0.37036800384521484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,2,128,1,fp8,fp8,0,0.37221864859263104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,4,128,1,float16,float16,0,0.28729067246119183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,4,128,1,float16,fp8,0,0.28702400128046673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,4,128,1,fp8,fp8,0,0.3720693190892537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,8,128,1,float16,float16,0,0.2916053334871928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,8,128,1,float16,fp8,0,0.2911253372828166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,40,8,128,1,fp8,fp8,0,0.3756479819615682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,40,128,1,float16,fp8,0,0.1572533349196116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,1,128,1,float16,float16,0,0.14699199795722961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,40,128,1,float16,float16,0,0.16126400232315063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,40,128,1,fp8,fp8,0,0.21059733629226685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,1,128,1,float16,fp8,0,0.1462613344192505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,1,128,1,fp8,fp8,0,0.19748266537984213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,2,128,1,float16,fp8,0,0.1473973294099172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,2,128,1,fp8,fp8,0,0.1983413298924764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,4,128,1,float16,float16,0,0.14871467153231302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,4,128,1,float16,fp8,0,0.1483573317527771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,2,128,1,float16,float16,0,0.1470186710357666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,4,128,1,fp8,fp8,0,0.1984106699625651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,8,128,1,float16,float16,0,0.1520639955997467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,8,128,1,float16,fp8,0,0.15200000007947287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,40,8,128,1,fp8,fp8,0,0.20100800196329752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,40,128,1,float16,float16,0,0.08517866333325703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,40,128,1,float16,fp8,0,0.08265600105126698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,1,128,1,float16,float16,0,0.07990933458010356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,1,128,1,float16,fp8,0,0.08085866769154866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,40,128,1,fp8,fp8,0,0.11546666423479716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,1,128,1,fp8,fp8,0,0.10710933804512024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,2,128,1,float16,float16,0,0.08017600079377492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,2,128,1,float16,fp8,0,0.08024533092975616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,2,128,1,fp8,fp8,0,0.10680533448855083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,4,128,1,float16,float16,0,0.08085333307584126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,4,128,1,float16,fp8,0,0.08191466828187306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,8,128,1,float16,float16,0,0.08237333099047343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,8,128,1,float16,fp8,0,0.08264000217119853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,8,128,1,fp8,fp8,0,0.11216533184051514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,40,4,128,1,fp8,fp8,0,0.10859200358390808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,40,128,1,float16,float16,0,0.047983999053637184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,40,128,1,float16,fp8,0,0.047040000557899475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,40,128,1,fp8,fp8,0,0.06345066428184509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,1,128,1,float16,float16,0,0.04868799944718679
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,1,128,1,float16,fp8,0,0.04770666857560476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,1,128,1,fp8,fp8,0,0.06165333092212677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,2,128,1,float16,fp8,0,0.04865066707134247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,2,128,1,fp8,fp8,0,0.06147199869155884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,4,128,1,float16,float16,0,0.047882666190465294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,2,128,1,float16,float16,0,0.047775998711586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,4,128,1,float16,fp8,0,0.04905066887537638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,4,128,1,fp8,fp8,0,0.06205866734186808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,8,128,1,float16,float16,0,0.0487360010544459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,8,128,1,fp8,fp8,0,0.06332266827424367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,40,8,128,1,float16,fp8,0,0.04860266546408335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,1,128,1,float16,float16,0,0.03203733265399933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,40,128,1,fp8,fp8,0,0.03933866570393244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,40,128,1,float16,float16,0,0.03141866624355316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,40,128,1,float16,fp8,0,0.031498665610949196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,1,128,1,float16,fp8,0,0.03254399945338567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,1,128,1,fp8,fp8,0,0.03817066550254822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,2,128,1,float16,float16,0,0.03229333211978277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,2,128,1,float16,fp8,0,0.032560000816980995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,4,128,1,float16,float16,0,0.032405334214369454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,2,128,1,fp8,fp8,0,0.03915733347336451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,8,128,1,float16,float16,0,0.03286399940649668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,4,128,1,fp8,fp8,0,0.03941333293914795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,8,128,1,float16,fp8,0,0.0330826664964358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,4,128,1,float16,fp8,0,0.03226666649182638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,40,8,128,1,fp8,fp8,0,0.03965866565704346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,40,128,1,float16,float16,0,0.02334933231274287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,40,128,1,float16,fp8,0,0.02359466751416524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,40,128,1,fp8,fp8,0,0.028938665986061096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,1,128,1,float16,fp8,0,0.02385599911212921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,1,128,1,float16,float16,0,0.02345066765944163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,2,128,1,float16,float16,0,0.024165332317352295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,2,128,1,float16,fp8,0,0.0239680012067159
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,2,128,1,fp8,fp8,0,0.028832000990708668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,1,128,1,fp8,fp8,0,0.028549333413441975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,4,128,1,float16,float16,0,0.023904000719388325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,4,128,1,float16,fp8,0,0.023999998966852825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,8,128,1,float16,float16,0,0.023973333338896435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,4,128,1,fp8,fp8,0,0.028538666665554047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,8,128,1,float16,fp8,0,0.024330665667851765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,40,8,128,1,fp8,fp8,0,0.028768000503381092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,40,128,1,float16,fp8,0,0.02025066688656807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,40,128,1,fp8,fp8,0,0.024085332949956257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,1,128,1,float16,float16,0,0.02033599962790807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,40,128,1,float16,float16,0,0.019834666202465694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,1,128,1,float16,fp8,0,0.02089066555102666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,1,128,1,fp8,fp8,0,0.023183998962243397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,2,128,1,float16,float16,0,0.020586666961510975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,2,128,1,float16,fp8,0,0.020527999848127365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,2,128,1,fp8,fp8,0,0.02290133386850357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,4,128,1,float16,float16,0,0.020037333170572918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,4,128,1,fp8,fp8,0,0.02346133440732956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,8,128,1,float16,float16,0,0.020714666694402695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,8,128,1,float16,fp8,0,0.021530665457248688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,4,128,1,float16,fp8,0,0.02102400114138921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,40,8,128,1,fp8,fp8,0,0.023029332359631855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,40,128,1,float16,float16,0,0.016879999389251072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,40,128,1,float16,fp8,0,0.0174346665541331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,40,128,1,fp8,fp8,0,0.01939733326435089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,1,128,1,float16,float16,0,0.01714133347074191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,1,128,1,float16,fp8,0,0.017722666263580322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,2,128,1,float16,float16,0,0.016938666502634685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,2,128,1,fp8,fp8,0,0.018661333868900936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,2,128,1,float16,fp8,0,0.017488000293572743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,4,128,1,float16,float16,0,0.01716800034046173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,1,128,1,fp8,fp8,0,0.018960000326236088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,4,128,1,fp8,fp8,0,0.018944000204404194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,4,128,1,float16,fp8,0,0.017114666601022083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,8,128,1,float16,float16,0,0.017130666722853977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,8,128,1,float16,fp8,0,0.01754666616519292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,40,8,128,1,fp8,fp8,0,0.019109333554903667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,40,128,1,float16,float16,0,0.01676799977819125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,40,128,1,fp8,fp8,0,0.018559999763965607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,1,128,1,float16,float16,0,0.016976000120242436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,40,128,1,float16,fp8,0,0.01701333373785019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,1,128,1,float16,fp8,0,0.016970666746298473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,1,128,1,fp8,fp8,0,0.018559999763965607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,2,128,1,float16,float16,0,0.01685333376129468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,2,128,1,float16,fp8,0,0.01752000053723653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,2,128,1,fp8,fp8,0,0.018687999496857326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,4,128,1,float16,float16,0,0.016997333616018295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,8,128,1,float16,float16,0,0.016832000265518825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,4,128,1,fp8,fp8,0,0.01868266612291336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,8,128,1,float16,fp8,0,0.017690667261679966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,8,128,1,fp8,fp8,0,0.018778666853904724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,40,4,128,1,float16,fp8,0,0.017045332739750545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,1,128,1,float16,fp8,0,33.49413808186849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,1,128,1,fp8,fp8,0,22.464683532714844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,1,128,1,float16,float16,0,33.125562032063804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,2,128,1,fp8,fp8,0,21.999041239420574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,2,128,1,float16,float16,0,33.273274739583336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,2,128,1,float16,fp8,0,33.585540771484375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,4,128,1,float16,float16,0,33.41759999593099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,4,128,1,float16,fp8,0,33.457557678222656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,4,128,1,fp8,fp8,0,22.585108439127605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,8,128,1,float16,float16,0,33.268330891927086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,32,128,1,float16,float16,0,16.832608540852863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,8,128,1,fp8,fp8,0,22.008628845214844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,32,8,128,1,float16,fp8,0,33.43020884195963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,32,128,1,float16,fp8,0,16.617029825846355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,1,128,1,float16,float16,0,17.357072194417317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,1,128,1,fp8,fp8,0,11.23153559366862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,1,128,1,float16,fp8,0,16.62462870279948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,2,128,1,float16,float16,0,16.749136606852215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,2,128,1,float16,fp8,0,16.76478449503581
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,2,128,1,fp8,fp8,0,11.056533813476562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,4,128,1,float16,float16,0,17.319124857584637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,4,128,1,float16,fp8,0,16.93121083577474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,32,128,1,fp8,fp8,0,11.282180786132812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,4,128,1,fp8,fp8,0,11.121205647786459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,8,128,1,float16,fp8,0,16.84387715657552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,8,128,1,float16,float16,0,16.87921651204427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,32,8,128,1,fp8,fp8,0,11.103691101074219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,1,128,1,float16,float16,0,8.370543797810873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,32,128,1,float16,float16,0,8.53599484761556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,1,128,1,float16,fp8,0,8.489807764689127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,1,128,1,fp8,fp8,0,5.555125554402669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,32,128,1,float16,fp8,0,8.473834355672201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,2,128,1,float16,float16,0,8.533642450968424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,2,128,1,float16,fp8,0,8.479546864827475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,32,128,1,fp8,fp8,0,5.662874857584636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,2,128,1,fp8,fp8,0,5.670960108439128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,4,128,1,float16,float16,0,8.339109420776367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,4,128,1,float16,fp8,0,8.498218536376953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,4,128,1,fp8,fp8,0,5.470602671305339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,8,128,1,fp8,fp8,0,5.588511784871419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,8,128,1,float16,float16,0,8.449951807657877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,32,8,128,1,float16,fp8,0,8.358842849731445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,1,128,1,float16,float16,0,4.212714513142903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,1,128,1,float16,fp8,0,4.256693204243978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,1,128,1,fp8,fp8,0,2.8645652135213218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,32,128,1,float16,float16,0,4.192480087280273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,2,128,1,float16,float16,0,4.12722651163737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,32,128,1,fp8,fp8,0,2.885237375895182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,32,128,1,float16,fp8,0,4.176368077596028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,2,128,1,float16,fp8,0,4.099520047505696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,2,128,1,fp8,fp8,0,2.8682400385538735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,4,128,1,float16,fp8,0,4.367295900980632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,4,128,1,float16,float16,0,4.188101450602214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,4,128,1,fp8,fp8,0,2.8720693588256836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,8,128,1,float16,float16,0,4.16597334543864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,8,128,1,fp8,fp8,0,2.8841492335001626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,32,8,128,1,float16,fp8,0,4.305290540059407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,1,128,1,float16,float16,0,19.349039713541668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,1,128,1,float16,fp8,0,19.525450388590496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,2,128,1,float16,float16,0,19.1527837117513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,2,128,1,fp8,fp8,0,13.053323109944662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,2,128,1,float16,fp8,0,19.130111694335938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,4,128,1,float16,float16,0,19.602420806884766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,1,128,1,fp8,fp8,0,12.842559814453125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,4,128,1,float16,fp8,0,19.353312174479168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,4,128,1,fp8,fp8,0,13.288416544596354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,8,128,1,float16,fp8,0,19.487098693847656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,8,128,1,fp8,fp8,0,13.06823984781901
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,32,8,128,1,float16,float16,0,19.40234120686849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,32,128,1,float16,float16,0,9.805967966715494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,32,128,1,float16,fp8,0,9.68939208984375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,1,128,1,float16,float16,0,9.83357302347819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,1,128,1,float16,fp8,0,9.806533177693685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,1,128,1,fp8,fp8,0,6.43284289042155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,2,128,1,float16,float16,0,9.638901392618815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,2,128,1,float16,fp8,0,9.676698684692383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,2,128,1,fp8,fp8,0,6.351253509521484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,32,128,1,fp8,fp8,0,6.664394378662109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,4,128,1,float16,float16,0,9.938389460245768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,4,128,1,fp8,fp8,0,6.417968114217122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,4,128,1,float16,fp8,0,9.799386978149414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,8,128,1,float16,float16,0,9.652277628580729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,8,128,1,fp8,fp8,0,6.430517196655273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,1,128,1,float16,float16,0,4.960463841756185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,32,8,128,1,float16,fp8,0,9.788917541503906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,32,128,1,float16,float16,0,4.692816098531087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,32,128,1,float16,fp8,0,4.94757874806722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,1,128,1,float16,fp8,0,4.731034596761067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,1,128,1,fp8,fp8,0,3.2408533096313477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,2,128,1,float16,float16,0,4.926671981811523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,2,128,1,float16,fp8,0,4.884847958882649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,32,128,1,fp8,fp8,0,3.3026612599690757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,2,128,1,fp8,fp8,0,3.2435146967569985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,4,128,1,float16,float16,0,4.875898679097493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,4,128,1,float16,fp8,0,4.640720049540202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,4,128,1,fp8,fp8,0,3.2531894048055015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,8,128,1,float16,float16,0,4.92515214284261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,8,128,1,fp8,fp8,0,3.2691307067871094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,32,8,128,1,float16,fp8,0,4.636490821838379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,1,128,1,float16,float16,0,2.454304059346517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,32,128,1,float16,float16,0,2.4648106892903647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,1,128,1,float16,fp8,0,2.433834711710612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,32,128,1,float16,fp8,0,2.4844746589660645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,1,128,1,fp8,fp8,0,1.7464906374613445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,32,128,1,fp8,fp8,0,1.7691680590311687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,2,128,1,float16,float16,0,2.46560001373291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,2,128,1,float16,fp8,0,2.4312639236450195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,2,128,1,fp8,fp8,0,1.7470080057779949
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,4,128,1,float16,float16,0,2.464325269063314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,4,128,1,fp8,fp8,0,1.7526079813639324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,4,128,1,float16,fp8,0,2.4853493372599282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,8,128,1,float16,float16,0,2.4656532605489097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,8,128,1,float16,fp8,0,2.4504213333129883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,32,8,128,1,fp8,fp8,0,1.7576053937276204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,1,128,1,float16,float16,0,13.82375971476237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,1,128,1,float16,fp8,0,13.667444864908854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,1,128,1,fp8,fp8,0,9.15497080485026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,2,128,1,float16,float16,0,14.013871510823568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,2,128,1,float16,fp8,0,13.660672505696615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,2,128,1,fp8,fp8,0,9.358298619588217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,4,128,1,float16,float16,0,13.702208201090494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,4,128,1,fp8,fp8,0,9.321418762207031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,4,128,1,float16,fp8,0,13.779930114746094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,8,128,1,float16,float16,0,13.826432545979818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,8,128,1,float16,fp8,0,13.993525187174479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,32,8,128,1,fp8,fp8,0,9.28102938334147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,1,128,1,float16,float16,0,6.855536142985026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,32,128,1,float16,float16,0,7.058645248413086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,32,128,1,float16,fp8,0,6.885973612467448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,1,128,1,float16,fp8,0,6.897375742594401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,1,128,1,fp8,fp8,0,4.558538754781087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,2,128,1,float16,float16,0,6.849760055541992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,2,128,1,float16,fp8,0,6.8530934651692705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,32,128,1,fp8,fp8,0,4.803872108459473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,2,128,1,fp8,fp8,0,4.546074549357097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,4,128,1,float16,float16,0,6.8711198170979815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,4,128,1,float16,fp8,0,6.660207748413086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,4,128,1,fp8,fp8,0,4.570618629455566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,1,128,1,float16,float16,0,3.4415413538614907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,8,128,1,float16,float16,0,6.756181081136067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,8,128,1,float16,fp8,0,6.930575688680013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,32,8,128,1,fp8,fp8,0,4.620986620585124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,32,128,1,float16,fp8,0,3.382453282674154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,32,128,1,fp8,fp8,0,2.426074663798014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,32,128,1,float16,float16,0,3.376373291015625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,1,128,1,float16,fp8,0,3.4407679239908853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,1,128,1,fp8,fp8,0,2.3650986353556314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,2,128,1,float16,float16,0,3.350895881652832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,2,128,1,float16,fp8,0,3.302218755086263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,2,128,1,fp8,fp8,0,2.3656533559163413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,4,128,1,float16,float16,0,3.391882578531901
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,4,128,1,float16,fp8,0,3.312533378601074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,4,128,1,fp8,fp8,0,2.379605293273926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,8,128,1,float16,float16,0,3.3889118830362954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,8,128,1,float16,fp8,0,3.3833440144856772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,32,8,128,1,fp8,fp8,0,2.384965260823568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,1,128,1,float16,float16,0,1.8117705980936687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,32,128,1,float16,float16,0,1.773311932881673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,32,128,1,float16,fp8,0,1.7639039357503254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,1,128,1,float16,fp8,0,1.7602933247884114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,1,128,1,fp8,fp8,0,1.2886239687601726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,2,128,1,float16,float16,0,1.7848052978515625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,2,128,1,float16,fp8,0,1.7915840148925781
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,2,128,1,fp8,fp8,0,1.2896426518758137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,4,128,1,float16,fp8,0,1.7674026489257812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,4,128,1,float16,float16,0,1.7871306737263997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,4,128,1,fp8,fp8,0,1.2909653186798096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,8,128,1,float16,float16,0,1.7910985946655273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,32,128,1,fp8,fp8,0,1.3212587038675945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,8,128,1,float16,fp8,0,1.8019839922587078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,32,8,128,1,fp8,fp8,0,1.298261324564616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,1,128,1,float16,fp8,0,17.81288019816081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,1,128,1,float16,float16,0,17.83617655436198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,1,128,1,fp8,fp8,0,12.25380833943685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,2,128,1,float16,float16,0,18.029380798339844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,2,128,1,float16,fp8,0,17.94063440958659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,2,128,1,fp8,fp8,0,12.70800526936849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,4,128,1,float16,float16,0,18.10534413655599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,4,128,1,float16,fp8,0,17.866235097249348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,4,128,1,fp8,fp8,0,12.289034525553385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,8,128,1,float16,float16,0,18.21942901611328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,8,128,1,float16,fp8,0,18.28013865152995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,32,128,1,float16,float16,0,9.363018671671549
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,32,8,128,1,fp8,fp8,0,12.471941630045572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,1,128,1,float16,float16,0,9.078346888224283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,32,128,1,float16,fp8,0,9.046501159667969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,1,128,1,fp8,fp8,0,6.177493413289388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,1,128,1,float16,fp8,0,9.124261220296225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,2,128,1,float16,float16,0,9.15390396118164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,2,128,1,float16,fp8,0,8.977941513061523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,2,128,1,fp8,fp8,0,6.136282602945964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,32,128,1,fp8,fp8,0,6.310085296630859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,4,128,1,float16,float16,0,9.026463826497396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,4,128,1,float16,fp8,0,9.101823806762695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,4,128,1,fp8,fp8,0,6.082880020141602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,8,128,1,float16,float16,0,9.257754643758139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,1,128,1,float16,float16,0,4.452586809794108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,8,128,1,float16,fp8,0,9.098704020182291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,32,128,1,float16,float16,0,4.604117393493652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,32,8,128,1,fp8,fp8,0,6.178378423055013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,32,128,1,float16,fp8,0,4.3598025639851885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,1,128,1,float16,fp8,0,4.338735898335774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,1,128,1,fp8,fp8,0,3.078842798868815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,32,128,1,fp8,fp8,0,3.197317441304525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,2,128,1,float16,float16,0,4.436517397562663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,2,128,1,float16,fp8,0,4.47764269510905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,2,128,1,fp8,fp8,0,3.087002754211426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,4,128,1,float16,float16,0,4.432175954182942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,4,128,1,float16,fp8,0,4.548032124837239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,4,128,1,fp8,fp8,0,3.097482681274414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,8,128,1,float16,float16,0,4.4086612065633135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,1,128,1,float16,float16,0,2.2453014055887857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,8,128,1,fp8,fp8,0,3.120933214823405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,1,128,1,float16,fp8,0,2.2099787394205728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,32,8,128,1,float16,fp8,0,4.457098642985026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,32,128,1,float16,float16,0,2.242448012034098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,32,128,1,float16,fp8,0,2.2301759719848633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,32,128,1,fp8,fp8,0,1.6779306729634602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,1,128,1,fp8,fp8,0,1.6206612586975098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,2,128,1,float16,float16,0,2.239685376485189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,2,128,1,float16,fp8,0,2.2114933331807456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,2,128,1,fp8,fp8,0,1.6239412625630696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,4,128,1,float16,float16,0,2.238719940185547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,4,128,1,float16,fp8,0,2.2127253214518228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,4,128,1,fp8,fp8,0,1.6322986284891765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,8,128,1,float16,float16,0,2.2597227096557617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,8,128,1,float16,fp8,0,2.225173314412435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,32,8,128,1,fp8,fp8,0,1.6390293439229329
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,1,128,1,float16,float16,0,1.2313439846038818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,32,128,1,float16,fp8,0,1.199514627456665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,32,128,1,float16,float16,0,1.2196533679962158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,32,128,1,fp8,fp8,0,0.9207733472188314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,1,128,1,float16,fp8,0,1.200927972793579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,1,128,1,fp8,fp8,0,0.8996480305989584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,2,128,1,float16,float16,0,1.2187360127766926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,2,128,1,float16,fp8,0,1.218394676844279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,2,128,1,fp8,fp8,0,0.8990879853566488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,4,128,1,float16,fp8,0,1.203114668528239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,4,128,1,float16,float16,0,1.2217493057250977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,4,128,1,fp8,fp8,0,0.9012746810913086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,8,128,1,float16,float16,0,1.222879966100057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,8,128,1,float16,fp8,0,1.206223964691162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,32,8,128,1,fp8,fp8,0,0.9078826904296875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,1,128,1,float16,fp8,0,10.562970479329428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,1,128,1,float16,float16,0,10.577509562174479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,1,128,1,fp8,fp8,0,7.399072011311849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,2,128,1,float16,fp8,0,10.627850850423178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,2,128,1,fp8,fp8,0,7.4205067952473955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,2,128,1,float16,float16,0,10.702645619710287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,4,128,1,float16,float16,0,10.772212982177734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,4,128,1,float16,fp8,0,10.629034678141275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,4,128,1,fp8,fp8,0,7.357472101847331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,8,128,1,fp8,fp8,0,7.468533198038737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,8,128,1,float16,fp8,0,10.803471883138021
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,32,8,128,1,float16,float16,0,10.779711405436197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,32,128,1,float16,float16,0,5.464282353719075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,32,128,1,float16,fp8,0,5.367669423421224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,1,128,1,float16,float16,0,5.130832036336263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,1,128,1,float16,fp8,0,5.074735959370931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,1,128,1,fp8,fp8,0,3.7076212565104165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,2,128,1,float16,float16,0,5.370730717976888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,2,128,1,float16,fp8,0,5.207285245259603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,2,128,1,fp8,fp8,0,3.720479965209961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,4,128,1,float16,float16,0,5.108981450398763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,32,128,1,fp8,fp8,0,3.8676745096842446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,4,128,1,float16,fp8,0,5.176074663798015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,4,128,1,fp8,fp8,0,3.7255732218424478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,8,128,1,float16,fp8,0,5.163039843241374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,1,128,1,float16,float16,0,2.575354735056559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,8,128,1,fp8,fp8,0,3.7654987970987954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,32,8,128,1,float16,float16,0,5.315221468607585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,32,128,1,float16,float16,0,2.6324853897094727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,1,128,1,float16,fp8,0,2.553365389506022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,1,128,1,fp8,fp8,0,1.9020959536234539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,32,128,1,float16,fp8,0,2.6151572863260903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,2,128,1,float16,float16,0,2.626522699991862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,32,128,1,fp8,fp8,0,2.002415974934896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,2,128,1,float16,fp8,0,2.550293286641439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,2,128,1,fp8,fp8,0,1.9061279296875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,4,128,1,float16,fp8,0,2.551813284556071
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,4,128,1,float16,float16,0,2.597424030303955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,4,128,1,fp8,fp8,0,1.9170400301615398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,8,128,1,float16,fp8,0,2.598207950592041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,8,128,1,float16,float16,0,2.6521013577779136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,32,8,128,1,fp8,fp8,0,1.931290626525879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,1,128,1,float16,float16,0,1.3613707224527996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,32,128,1,float16,float16,0,1.3750613530476887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,1,128,1,float16,fp8,0,1.3401546478271484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,1,128,1,fp8,fp8,0,1.0174453258514404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,2,128,1,float16,float16,0,1.3682559331258137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,32,128,1,fp8,fp8,0,1.063109318415324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,32,128,1,float16,fp8,0,1.3655625979105632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,2,128,1,float16,fp8,0,1.3431466420491536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,2,128,1,fp8,fp8,0,1.0178666909535725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,4,128,1,float16,float16,0,1.3661012649536133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,4,128,1,float16,fp8,0,1.3658666610717773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,8,128,1,float16,float16,0,1.377413272857666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,8,128,1,float16,fp8,0,1.3522027333577473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,4,128,1,fp8,fp8,0,1.0226613680521648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,32,8,128,1,fp8,fp8,0,1.0294240315755208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,32,128,1,float16,float16,0,0.7578826745351156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,32,128,1,float16,fp8,0,0.7512693405151367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,1,128,1,float16,float16,0,0.7592053413391113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,1,128,1,float16,fp8,0,0.7460107008616129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,32,128,1,fp8,fp8,0,0.5738933483759562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,1,128,1,fp8,fp8,0,0.5457280079523722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,2,128,1,float16,float16,0,0.7599786917368571
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,2,128,1,float16,fp8,0,0.7458879947662354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,2,128,1,fp8,fp8,0,0.5459413528442383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,4,128,1,float16,float16,0,0.7619946797688802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,4,128,1,float16,fp8,0,0.7524000008900961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,4,128,1,fp8,fp8,0,0.5485546588897705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,8,128,1,float16,fp8,0,0.7500426769256592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,8,128,1,float16,float16,0,0.7653813362121582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,32,8,128,1,fp8,fp8,0,0.5534986654917399
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,1,128,1,float16,float16,0,10.449082692464193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,1,128,1,fp8,fp8,0,7.477754592895508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,1,128,1,float16,fp8,0,10.209599812825521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,2,128,1,float16,fp8,0,10.254063924153646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,2,128,1,fp8,fp8,0,7.50921630859375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,2,128,1,float16,float16,0,10.564709345499674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,4,128,1,float16,float16,0,10.3210080464681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,4,128,1,float16,fp8,0,10.550538380940756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,4,128,1,fp8,fp8,0,7.541274388631185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,8,128,1,float16,float16,0,10.587823867797852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,8,128,1,float16,fp8,0,10.373029073079428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,32,8,128,1,fp8,fp8,0,7.623717625935872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,32,128,1,float16,float16,0,5.433584213256836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,32,128,1,float16,fp8,0,5.2071787516276045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,1,128,1,float16,float16,0,5.068106651306152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,1,128,1,float16,fp8,0,4.872735977172852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,1,128,1,fp8,fp8,0,3.7732534408569336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,2,128,1,float16,float16,0,5.085413297017415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,2,128,1,fp8,fp8,0,3.7812105814615884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,2,128,1,float16,fp8,0,5.125445365905762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,4,128,1,float16,float16,0,5.06608517964681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,32,128,1,fp8,fp8,0,4.010992050170898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,4,128,1,float16,fp8,0,4.978768030802409
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,4,128,1,fp8,fp8,0,3.799824078877767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,8,128,1,float16,float16,0,5.140432039896647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,8,128,1,float16,fp8,0,5.021674791971843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,32,8,128,1,fp8,fp8,0,3.840026537577311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,32,128,1,float16,float16,0,2.616373380025228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,1,128,1,float16,float16,0,2.487002690633138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,32,128,1,float16,fp8,0,2.5811732610066733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,1,128,1,float16,fp8,0,2.4524052937825522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,1,128,1,fp8,fp8,0,1.9106559753417969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,2,128,1,float16,float16,0,2.5324427286783853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,2,128,1,float16,fp8,0,2.449471950531006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,32,128,1,fp8,fp8,0,2.0497546195983887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,2,128,1,fp8,fp8,0,1.9179040590922039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,4,128,1,float16,float16,0,2.504848003387451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,4,128,1,float16,fp8,0,2.462202707926432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,4,128,1,fp8,fp8,0,1.928160031636556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,8,128,1,float16,fp8,0,2.477226734161377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,8,128,1,fp8,fp8,0,1.9535892804463704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,32,128,1,float16,float16,0,1.339237372080485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,32,8,128,1,float16,float16,0,2.561855951944987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,1,128,1,float16,float16,0,1.2940320173899333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,1,128,1,float16,fp8,0,1.271674633026123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,32,128,1,float16,fp8,0,1.3247360388437908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,1,128,1,fp8,fp8,0,1.0002453327178955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,2,128,1,float16,float16,0,1.2984639803568523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,2,128,1,float16,fp8,0,1.2899786631266277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,32,128,1,fp8,fp8,0,1.0758026440938313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,2,128,1,fp8,fp8,0,1.003386656443278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,4,128,1,float16,fp8,0,1.279584010442098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,4,128,1,float16,float16,0,1.3021492958068848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,4,128,1,fp8,fp8,0,1.0102880001068115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,8,128,1,float16,float16,0,1.3088800112406414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,8,128,1,float16,fp8,0,1.3044853210449219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,1,128,1,float16,float16,0,0.7009387016296387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,32,8,128,1,fp8,fp8,0,1.0235466957092285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,1,128,1,float16,fp8,0,0.6882239977518717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,32,128,1,float16,float16,0,0.7145226796468099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,1,128,1,fp8,fp8,0,0.5483520030975342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,32,128,1,float16,fp8,0,0.7068906625111898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,32,128,1,fp8,fp8,0,0.5821813344955444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,2,128,1,float16,float16,0,0.7039093176523844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,2,128,1,float16,fp8,0,0.6915679772694906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,2,128,1,fp8,fp8,0,0.5495680173238119
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,4,128,1,float16,fp8,0,0.6949439843495687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,4,128,1,fp8,fp8,0,0.5530133247375488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,4,128,1,float16,float16,0,0.7051839828491211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,8,128,1,float16,float16,0,0.7085866928100586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,8,128,1,float16,fp8,0,0.695845365524292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,32,8,128,1,fp8,fp8,0,0.5580373207728068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,32,128,1,float16,float16,0,0.4095573425292969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,1,128,1,float16,fp8,0,0.39689067999521893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,32,128,1,float16,fp8,0,0.406607985496521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,32,128,1,fp8,fp8,0,0.32021866242090863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,1,128,1,fp8,fp8,0,0.30189333359400433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,1,128,1,float16,float16,0,0.4041759967803955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,2,128,1,float16,float16,0,0.40650665760040283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,2,128,1,float16,fp8,0,0.39766399065653485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,2,128,1,fp8,fp8,0,0.30321067571640015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,4,128,1,float16,fp8,0,0.3999840021133423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,4,128,1,fp8,fp8,0,0.30458666880925495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,4,128,1,float16,float16,0,0.4085226853688558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,8,128,1,float16,float16,0,0.40858133633931476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,8,128,1,float16,fp8,0,0.4038506746292114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,32,8,128,1,fp8,fp8,0,0.30644800265630084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,1,128,1,float16,float16,0,6.158954620361328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,1,128,1,float16,fp8,0,6.119210561116536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,1,128,1,fp8,fp8,0,4.789178530375163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,2,128,1,float16,fp8,0,6.06550407409668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,2,128,1,fp8,fp8,0,4.805551846822103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,2,128,1,float16,float16,0,6.380591710408528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,4,128,1,float16,float16,0,6.247445424397786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,4,128,1,float16,fp8,0,6.131893157958984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,4,128,1,fp8,fp8,0,4.831141471862793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,8,128,1,float16,float16,0,6.393114725748698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,8,128,1,float16,fp8,0,6.176778793334961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,32,8,128,1,fp8,fp8,0,4.901642799377441
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,32,128,1,float16,float16,0,3.336085319519043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,1,128,1,float16,float16,0,3.0477708180745444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,32,128,1,float16,fp8,0,3.172133445739746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,1,128,1,float16,fp8,0,2.999706586201986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,1,128,1,fp8,fp8,0,2.4123892784118652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,2,128,1,float16,float16,0,3.048661231994629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,2,128,1,fp8,fp8,0,2.4207092920939126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,2,128,1,float16,fp8,0,3.06114133199056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,4,128,1,float16,float16,0,3.0610507329305015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,32,128,1,fp8,fp8,0,2.6123627026875815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,4,128,1,float16,fp8,0,3.002650578816732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,4,128,1,fp8,fp8,0,2.4335145950317383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,8,128,1,float16,float16,0,3.10697078704834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,8,128,1,float16,fp8,0,3.0236905415852866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,1,128,1,float16,float16,0,1.5493653615315754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,32,128,1,float16,float16,0,1.6447733243306477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,1,128,1,float16,fp8,0,1.5202080408732097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,32,8,128,1,fp8,fp8,0,2.4688746134440103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,32,128,1,float16,fp8,0,1.619765281677246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,32,128,1,fp8,fp8,0,1.3428106307983398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,1,128,1,fp8,fp8,0,1.2324693202972412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,2,128,1,fp8,fp8,0,1.237930695215861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,2,128,1,float16,fp8,0,1.5196266174316406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,2,128,1,float16,float16,0,1.5528213183085124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,4,128,1,float16,fp8,0,1.527440071105957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,4,128,1,float16,float16,0,1.575727939605713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,4,128,1,fp8,fp8,0,1.2435786724090576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,8,128,1,float16,float16,0,1.5716320673624675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,8,128,1,float16,fp8,0,1.5460319519042969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,32,8,128,1,fp8,fp8,0,1.2596746285756428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,1,128,1,float16,float16,0,0.8172106742858887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,32,128,1,float16,float16,0,0.8539573351542155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,32,128,1,float16,fp8,0,0.8415306409200033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,1,128,1,float16,fp8,0,0.7989439964294434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,32,128,1,fp8,fp8,0,0.7121493021647135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,1,128,1,fp8,fp8,0,0.6562079985936483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,2,128,1,float16,float16,0,0.8193600177764893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,2,128,1,float16,fp8,0,0.8020426432291666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,4,128,1,float16,float16,0,0.822165330251058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,2,128,1,fp8,fp8,0,0.6573546727498373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,4,128,1,float16,fp8,0,0.8065919876098633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,4,128,1,fp8,fp8,0,0.6597333351771036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,8,128,1,float16,float16,0,0.8277280330657959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,8,128,1,fp8,fp8,0,0.668831984202067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,32,8,128,1,float16,fp8,0,0.8117333253224691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,32,128,1,float16,float16,0,0.4646186828613281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,1,128,1,float16,float16,0,0.4517013231913249
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,32,128,1,fp8,fp8,0,0.382368008295695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,32,128,1,float16,fp8,0,0.45710933208465576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,1,128,1,float16,fp8,0,0.44248000780741376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,1,128,1,fp8,fp8,0,0.34905068079630536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,2,128,1,float16,float16,0,0.4525973399480184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,2,128,1,float16,fp8,0,0.4429173469543457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,4,128,1,float16,float16,0,0.45476265748341876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,4,128,1,float16,fp8,0,0.44401601950327557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,2,128,1,fp8,fp8,0,0.3495839834213257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,4,128,1,fp8,fp8,0,0.35226666927337646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,8,128,1,float16,float16,0,0.45715733369191486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,8,128,1,float16,fp8,0,0.4474826653798421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,32,8,128,1,fp8,fp8,0,0.3567519982655843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,32,128,1,float16,float16,0,0.2547253370285034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,1,128,1,float16,float16,0,0.24314665794372559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,1,128,1,float16,fp8,0,0.23651733001073202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,32,128,1,float16,fp8,0,0.2503040035565694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,1,128,1,fp8,fp8,0,0.20411733786265054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,32,128,1,fp8,fp8,0,0.22033067544301352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,2,128,1,float16,float16,0,0.24369599421819052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,2,128,1,fp8,fp8,0,0.2056586742401123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,2,128,1,float16,fp8,0,0.23677333196004233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,4,128,1,float16,float16,0,0.24474134047826132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,4,128,1,float16,fp8,0,0.2376319964726766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,4,128,1,fp8,fp8,0,0.2068106730779012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,8,128,1,float16,fp8,0,0.24005866050720215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,8,128,1,fp8,fp8,0,0.20891733964284262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,32,8,128,1,float16,float16,0,0.246943990389506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,1,128,1,float16,fp8,0,6.204938888549805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,1,128,1,fp8,fp8,0,5.304128011067708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,2,128,1,float16,fp8,0,6.224954605102539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,2,128,1,float16,float16,0,6.5134932200113935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,1,128,1,float16,float16,0,6.533407847086589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,2,128,1,fp8,fp8,0,5.3156585693359375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,4,128,1,float16,float16,0,6.640602747599284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,4,128,1,float16,fp8,0,6.267215728759766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,4,128,1,fp8,fp8,0,5.344933191935222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,8,128,1,float16,fp8,0,6.386175791422526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,8,128,1,fp8,fp8,0,5.42581303914388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,32,8,128,1,float16,float16,0,6.603098551432292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,32,128,1,float16,float16,0,3.49942938486735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,1,128,1,float16,float16,0,3.194234530131022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,32,128,1,float16,fp8,0,3.4332907994588218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,1,128,1,float16,fp8,0,3.079941431681315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,1,128,1,fp8,fp8,0,2.639904022216797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,2,128,1,float16,float16,0,3.203872044881185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,2,128,1,float16,fp8,0,3.093418757120768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,2,128,1,fp8,fp8,0,2.6526400248209634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,4,128,1,float16,float16,0,3.2496906916300454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,32,128,1,fp8,fp8,0,2.912245432535807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,4,128,1,float16,fp8,0,3.1103893915812173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,4,128,1,fp8,fp8,0,2.669370651245117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,8,128,1,float16,float16,0,3.291050593058268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,1,128,1,float16,float16,0,1.5904587109883626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,8,128,1,float16,fp8,0,3.152048110961914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,32,128,1,float16,float16,0,1.7714932759602864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,1,128,1,float16,fp8,0,1.5495519638061523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,32,128,1,float16,fp8,0,1.7143306732177734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,1,128,1,fp8,fp8,0,1.3266080220540364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,2,128,1,float16,float16,0,1.598090648651123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,32,8,128,1,fp8,fp8,0,2.7100960413614907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,32,128,1,fp8,fp8,0,1.4795786539713542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,2,128,1,float16,fp8,0,1.5620479583740234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,2,128,1,fp8,fp8,0,1.335477352142334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,4,128,1,float16,float16,0,1.605434735616048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,4,128,1,float16,fp8,0,1.5620533625284831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,4,128,1,fp8,fp8,0,1.347365379333496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,8,128,1,float16,float16,0,1.6214186350504558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,8,128,1,float16,fp8,0,1.5808906555175781
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,1,128,1,float16,float16,0,0.8270026842753092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,32,8,128,1,fp8,fp8,0,1.366410732269287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,32,128,1,float16,float16,0,0.8946613470713297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,1,128,1,float16,fp8,0,0.8038240273793539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,1,128,1,fp8,fp8,0,0.6913920243581136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,32,128,1,float16,fp8,0,0.8789013226826986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,32,128,1,fp8,fp8,0,0.7695146401723226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,2,128,1,float16,float16,0,0.8316693305969238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,2,128,1,float16,fp8,0,0.8094080289204916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,2,128,1,fp8,fp8,0,0.695909341176351
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,4,128,1,float16,fp8,0,0.8114666938781738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,4,128,1,fp8,fp8,0,0.6991946697235107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,4,128,1,float16,float16,0,0.8330132961273193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,8,128,1,float16,float16,0,0.8422186374664307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,8,128,1,fp8,fp8,0,0.7104533513387045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,32,8,128,1,float16,fp8,0,0.8243306477864584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,1,128,1,float16,fp8,0,0.4327840010325114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,32,128,1,float16,float16,0,0.473413348197937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,1,128,1,float16,float16,0,0.44307200113932294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,32,128,1,float16,fp8,0,0.4633653163909912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,32,128,1,fp8,fp8,0,0.4151466687520345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,1,128,1,fp8,fp8,0,0.3753066857655843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,2,128,1,float16,float16,0,0.44762667020161945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,2,128,1,float16,fp8,0,0.43507198492685956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,2,128,1,fp8,fp8,0,0.3783200184504191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,4,128,1,float16,fp8,0,0.4374613364537557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,4,128,1,fp8,fp8,0,0.3790080149968465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,4,128,1,float16,float16,0,0.44705601533253986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,8,128,1,float16,float16,0,0.4529600143432617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,8,128,1,float16,fp8,0,0.4429226716359456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,32,8,128,1,fp8,fp8,0,0.3844746748606364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,1,128,1,float16,fp8,0,0.2456373373667399
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,1,128,1,fp8,fp8,0,0.20383467276891074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,32,128,1,float16,fp8,0,0.25969066222508747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,32,128,1,float16,float16,0,0.2648853262265523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,1,128,1,float16,float16,0,0.251418670018514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,32,128,1,fp8,fp8,0,0.22638400395711264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,2,128,1,float16,float16,0,0.25336533784866333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,2,128,1,float16,fp8,0,0.24748265743255615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,2,128,1,fp8,fp8,0,0.20497065782546997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,4,128,1,float16,float16,0,0.2552586595217387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,4,128,1,float16,fp8,0,0.24964799483617148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,8,128,1,float16,float16,0,0.25863466660181683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,4,128,1,fp8,fp8,0,0.20578134059906006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,8,128,1,float16,fp8,0,0.25146667162577313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,32,8,128,1,fp8,fp8,0,0.2088373303413391
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,32,128,1,float16,float16,0,0.14945600430170694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,1,128,1,float16,float16,0,0.13899733622868857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,32,128,1,float16,fp8,0,0.14600533246994019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,32,128,1,fp8,fp8,0,0.13488533099492392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,1,128,1,fp8,fp8,0,0.12258133292198181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,1,128,1,float16,fp8,0,0.13666666547457376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,2,128,1,float16,float16,0,0.1386666695276896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,2,128,1,float16,fp8,0,0.13590400417645773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,2,128,1,fp8,fp8,0,0.12319999933242798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,4,128,1,float16,float16,0,0.14017066359519958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,4,128,1,float16,fp8,0,0.13659200072288513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,4,128,1,fp8,fp8,0,0.12548266847928366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,8,128,1,float16,fp8,0,0.13784000277519226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,8,128,1,float16,float16,0,0.14100266496340433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,32,8,128,1,fp8,fp8,0,0.12716266512870789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,1,128,1,float16,float16,0,4.203685442606608
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,1,128,1,float16,fp8,0,4.003733317057292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,1,128,1,fp8,fp8,0,3.5786027908325195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,2,128,1,float16,float16,0,4.202698707580566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,2,128,1,float16,fp8,0,4.07479985555013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,2,128,1,fp8,fp8,0,3.571440060933431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,4,128,1,float16,float16,0,4.237573305765788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,4,128,1,float16,fp8,0,4.037391980489095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,4,128,1,fp8,fp8,0,3.598416010538737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,8,128,1,float16,float16,0,4.308229446411133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,8,128,1,float16,fp8,0,4.087663968404134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,32,128,1,float16,float16,0,2.3079253832499185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,32,8,128,1,fp8,fp8,0,3.646202723185221
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,1,128,1,float16,float16,0,2.0669333140055337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,32,128,1,float16,fp8,0,2.2281813621520996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,1,128,1,float16,fp8,0,1.9827307065327961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,1,128,1,fp8,fp8,0,1.7706880569458008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,2,128,1,float16,float16,0,2.068005402882894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,2,128,1,float16,fp8,0,1.9967573483784993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,2,128,1,fp8,fp8,0,1.7790026664733887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,4,128,1,float16,float16,0,2.0898027420043945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,32,128,1,fp8,fp8,0,1.990810712178548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,4,128,1,fp8,fp8,0,1.7929387092590332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,4,128,1,float16,fp8,0,2.001471996307373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,8,128,1,float16,float16,0,2.1232266426086426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,1,128,1,float16,float16,0,1.0384746392567952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,8,128,1,float16,fp8,0,2.040757338205973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,32,8,128,1,fp8,fp8,0,1.824992020924886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,32,128,1,float16,float16,0,1.1616106828053792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,32,128,1,float16,fp8,0,1.1328852971394856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,32,128,1,fp8,fp8,0,1.0172053178151448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,2,128,1,float16,float16,0,1.0429333051045735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,2,128,1,float16,fp8,0,1.0101173718770344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,1,128,1,fp8,fp8,0,0.8999573389689127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,1,128,1,float16,fp8,0,1.0054453214009602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,2,128,1,fp8,fp8,0,0.9040053685506185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,4,128,1,float16,float16,0,1.0479893684387207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,4,128,1,float16,fp8,0,1.0168639818827312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,4,128,1,fp8,fp8,0,0.913103977839152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,32,128,1,float16,float16,0,0.6019733349482218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,8,128,1,float16,float16,0,1.0606186389923096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,8,128,1,float16,fp8,0,1.030901352564494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,32,8,128,1,fp8,fp8,0,0.9270453453063965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,1,128,1,float16,float16,0,0.5441173315048218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,32,128,1,float16,fp8,0,0.585423986117045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,1,128,1,fp8,fp8,0,0.47273067633310956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,1,128,1,float16,fp8,0,0.5294880072275797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,32,128,1,fp8,fp8,0,0.5333546797434489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,2,128,1,float16,float16,0,0.5483200152715048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,2,128,1,fp8,fp8,0,0.47465598583221436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,2,128,1,float16,fp8,0,0.5306880076726278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,4,128,1,float16,float16,0,0.5504853328069051
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,4,128,1,float16,fp8,0,0.5346133311589559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,4,128,1,fp8,fp8,0,0.47990934054056805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,8,128,1,float16,float16,0,0.5564853350321451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,8,128,1,fp8,fp8,0,0.4869866768519084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,32,128,1,float16,fp8,0,0.31570667028427124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,1,128,1,float16,float16,0,0.2982293367385864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,32,8,128,1,float16,fp8,0,0.541312019030253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,32,128,1,float16,float16,0,0.3216000000635783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,32,128,1,fp8,fp8,0,0.2977919975916545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,1,128,1,fp8,fp8,0,0.25126399596532184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,1,128,1,float16,fp8,0,0.28918399413426715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,2,128,1,float16,float16,0,0.300437331199646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,2,128,1,float16,fp8,0,0.29313067595163983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,2,128,1,fp8,fp8,0,0.2537386616071065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,4,128,1,float16,float16,0,0.30107732613881427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,4,128,1,float16,fp8,0,0.2933173378308614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,4,128,1,fp8,fp8,0,0.25456533829371136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,8,128,1,float16,float16,0,0.30486400922139484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,8,128,1,float16,fp8,0,0.29712533950805664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,32,8,128,1,fp8,fp8,0,0.2604213356971741
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,1,128,1,float16,float16,0,0.16270400087038675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,1,128,1,float16,fp8,0,0.15797332922617593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,32,128,1,float16,float16,0,0.1844159960746765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,32,128,1,fp8,fp8,0,0.16032532850901285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,32,128,1,float16,fp8,0,0.18086934089660645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,1,128,1,fp8,fp8,0,0.14429866274197897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,2,128,1,float16,float16,0,0.16385066509246826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,2,128,1,float16,fp8,0,0.15913066267967224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,2,128,1,fp8,fp8,0,0.1451573371887207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,4,128,1,float16,fp8,0,0.16014933586120605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,8,128,1,float16,float16,0,0.16671466827392578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,4,128,1,fp8,fp8,0,0.14689600467681885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,4,128,1,float16,float16,0,0.16386666893959045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,8,128,1,float16,fp8,0,0.16292267044385275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,32,8,128,1,fp8,fp8,0,0.14816000064214072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,32,128,1,float16,float16,0,0.10508267084757487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,1,128,1,float16,float16,0,0.09755733609199524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,32,128,1,fp8,fp8,0,0.09879466891288757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,32,128,1,float16,fp8,0,0.10389866431554158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,1,128,1,fp8,fp8,0,0.08892800410588582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,1,128,1,float16,fp8,0,0.09609599908192952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,2,128,1,float16,fp8,0,0.0960746705532074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,2,128,1,float16,float16,0,0.09755733609199524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,4,128,1,float16,float16,0,0.0979253351688385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,2,128,1,fp8,fp8,0,0.08798933029174805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,4,128,1,float16,fp8,0,0.09640000263849895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,4,128,1,fp8,fp8,0,0.09088533123334248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,8,128,1,float16,fp8,0,0.09758399923642476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,8,128,1,fp8,fp8,0,0.09322133660316467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,32,8,128,1,float16,float16,0,0.09943999846776326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,1,128,1,float16,float16,0,3.833786646525065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,1,128,1,fp8,fp8,0,3.4693705240885415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,2,128,1,float16,fp8,0,3.9470720291137695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,2,128,1,float16,float16,0,3.9213654200236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,1,128,1,float16,fp8,0,3.8152478535970054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,2,128,1,fp8,fp8,0,3.5782292683919272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,4,128,1,float16,float16,0,3.9868319829305015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,4,128,1,float16,fp8,0,4.017573356628418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,4,128,1,fp8,fp8,0,3.6357545852661133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,8,128,1,float16,fp8,0,4.1819413503011065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,8,128,1,float16,float16,0,4.136410713195801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,32,128,1,float16,float16,0,2.5324320793151855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,32,8,128,1,fp8,fp8,0,3.790805180867513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,1,128,1,float16,float16,0,1.9224905967712402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,1,128,1,float16,fp8,0,1.9243787129720051
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,1,128,1,fp8,fp8,0,1.7273653348286946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,32,128,1,float16,fp8,0,2.418549378712972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,2,128,1,float16,float16,0,1.9523040453592937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,32,128,1,fp8,fp8,0,2.1380747159322104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,2,128,1,fp8,fp8,0,1.7833919525146484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,2,128,1,float16,fp8,0,1.9645013809204102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,4,128,1,float16,float16,0,1.9803093274434407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,4,128,1,float16,fp8,0,1.9991893768310547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,4,128,1,fp8,fp8,0,1.8237120310465496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,8,128,1,float16,float16,0,2.0303893089294434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,1,128,1,float16,float16,0,0.9785866737365723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,8,128,1,float16,fp8,0,2.0638346672058105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,32,8,128,1,fp8,fp8,0,1.8965226809183757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,32,128,1,float16,float16,0,1.261519988377889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,32,128,1,fp8,fp8,0,1.0791626771291096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,1,128,1,float16,fp8,0,0.9775893688201904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,32,128,1,float16,fp8,0,1.2209973335266113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,1,128,1,fp8,fp8,0,0.8645066420237223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,2,128,1,float16,float16,0,0.9899146556854248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,4,128,1,float16,float16,0,0.9989066918691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,2,128,1,float16,fp8,0,0.9952053229014078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,4,128,1,float16,fp8,0,1.0017546812693279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,4,128,1,fp8,fp8,0,0.9234933058420817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,2,128,1,fp8,fp8,0,0.8946080207824707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,8,128,1,float16,float16,0,1.022330681482951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,8,128,1,float16,fp8,0,1.0216373602549236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,1,128,1,float16,float16,0,0.5040106773376465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,1,128,1,float16,fp8,0,0.5040533145268759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,32,8,128,1,fp8,fp8,0,0.9526613553365072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,1,128,1,fp8,fp8,0,0.4410880009333293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,32,128,1,float16,fp8,0,0.615226666132609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,32,128,1,fp8,fp8,0,0.5502986510594686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,32,128,1,float16,float16,0,0.6441546678543091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,2,128,1,float16,fp8,0,0.5113333463668823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,2,128,1,float16,float16,0,0.5088373422622681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,2,128,1,fp8,fp8,0,0.4562720060348511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,4,128,1,float16,float16,0,0.5136693318684896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,4,128,1,float16,fp8,0,0.5143306652704874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,4,128,1,fp8,fp8,0,0.4686239957809448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,8,128,1,float16,float16,0,0.5269013245900472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,1,128,1,float16,float16,0,0.2682346701622009
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,8,128,1,float16,fp8,0,0.5252480109532675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,32,128,1,float16,float16,0,0.3333546717961629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,32,128,1,float16,fp8,0,0.32200533151626587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,32,8,128,1,fp8,fp8,0,0.4872906605402629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,1,128,1,fp8,fp8,0,0.23459200064341226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,32,128,1,fp8,fp8,0,0.2823893427848816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,1,128,1,float16,fp8,0,0.26708267132441205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,2,128,1,float16,float16,0,0.27010132869084674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,2,128,1,float16,fp8,0,0.27027199665705365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,2,128,1,fp8,fp8,0,0.24278400341669717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,4,128,1,float16,fp8,0,0.272816002368927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,4,128,1,float16,float16,0,0.27135467529296875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,4,128,1,fp8,fp8,0,0.2424479921658834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,8,128,1,float16,float16,0,0.27804799874623615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,8,128,1,float16,fp8,0,0.277344008286794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,32,8,128,1,fp8,fp8,0,0.2534719904263814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,32,128,1,float16,float16,0,0.1809119979540507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,1,128,1,float16,float16,0,0.14622933665911356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,32,128,1,float16,fp8,0,0.17538134256998697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,1,128,1,float16,fp8,0,0.14629333217938742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,1,128,1,fp8,fp8,0,0.12359999616940816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,2,128,1,float16,float16,0,0.14812800288200378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,2,128,1,float16,fp8,0,0.14893866578737894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,32,128,1,fp8,fp8,0,0.14877866705258688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,2,128,1,fp8,fp8,0,0.12583466370900473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,4,128,1,float16,float16,0,0.15102400382359824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,4,128,1,float16,fp8,0,0.14967999855677286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,4,128,1,fp8,fp8,0,0.1269760032494863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,8,128,1,float16,float16,0,0.15387733777364096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,8,128,1,float16,fp8,0,0.15383467078208923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,32,128,1,float16,float16,0,0.0995253324508667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,32,8,128,1,fp8,fp8,0,0.13035733501116434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,1,128,1,float16,float16,0,0.07871466875076294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,32,128,1,float16,fp8,0,0.0981066624323527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,1,128,1,float16,fp8,0,0.07918933530648549
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,1,128,1,fp8,fp8,0,0.06898666421572368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,32,128,1,fp8,fp8,0,0.08407466610272725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,2,128,1,float16,float16,0,0.07961066563924153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,2,128,1,fp8,fp8,0,0.07036266724268596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,2,128,1,float16,fp8,0,0.07995733122030894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,4,128,1,float16,fp8,0,0.08142933249473572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,4,128,1,float16,float16,0,0.08078399797280629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,4,128,1,fp8,fp8,0,0.07121066749095917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,8,128,1,float16,fp8,0,0.08247466882069905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,8,128,1,float16,float16,0,0.08298666775226593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,32,8,128,1,fp8,fp8,0,0.07500266532103221
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,1,128,1,float16,float16,0,0.0462773342927297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,32,128,1,fp8,fp8,0,0.05197333296140035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,32,128,1,float16,fp8,0,0.05323733389377594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,1,128,1,float16,fp8,0,0.04642133414745331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,32,128,1,float16,float16,0,0.05471999943256378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,1,128,1,fp8,fp8,0,0.04130133241415024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,2,128,1,float16,float16,0,0.046495998899141945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,2,128,1,float16,fp8,0,0.046240001916885376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,2,128,1,fp8,fp8,0,0.041519999504089355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,4,128,1,float16,fp8,0,0.046869332591692604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,4,128,1,float16,float16,0,0.046725332736968994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,4,128,1,fp8,fp8,0,0.041989331444104515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,8,128,1,float16,fp8,0,0.047242666284243263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,8,128,1,float16,float16,0,0.047279998660087585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,32,8,128,1,fp8,fp8,0,0.043162668744723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,1,128,1,float16,float16,0,2.9629761377970376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,1,128,1,float16,fp8,0,2.9622987111409507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,1,128,1,fp8,fp8,0,2.878335952758789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,2,128,1,float16,float16,0,3.070608139038086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,2,128,1,float16,fp8,0,3.0727945963541665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,4,128,1,float16,float16,0,3.116959889729818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,2,128,1,fp8,fp8,0,2.9556798934936523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,4,128,1,float16,fp8,0,3.1610399881998696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,4,128,1,fp8,fp8,0,3.0303306579589844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,8,128,1,float16,float16,0,3.285930633544922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,8,128,1,float16,fp8,0,3.287226676940918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,32,128,1,float16,float16,0,2.098698616027832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,32,8,128,1,fp8,fp8,0,3.1982399622599282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,1,128,1,float16,float16,0,1.4918452898661296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,1,128,1,float16,fp8,0,1.4918400446573894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,32,128,1,float16,fp8,0,2.0201600392659507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,1,128,1,fp8,fp8,0,1.4339466094970703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,2,128,1,float16,float16,0,1.5238293011983235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,2,128,1,float16,fp8,0,1.5410827000935872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,2,128,1,fp8,fp8,0,1.4728800455729167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,32,128,1,fp8,fp8,0,1.8482613563537598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,4,128,1,float16,float16,0,1.5505280494689941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,4,128,1,float16,fp8,0,1.5722079277038574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,4,128,1,fp8,fp8,0,1.5206185976664226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,8,128,1,float16,float16,0,1.6323946317036946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,8,128,1,float16,fp8,0,1.6317599614461262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,1,128,1,float16,float16,0,0.7592480182647705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,32,8,128,1,fp8,fp8,0,1.5915466944376628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,1,128,1,float16,fp8,0,0.7603466510772705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,1,128,1,fp8,fp8,0,0.7100106875101725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,32,128,1,float16,float16,0,1.0607253710428874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,32,128,1,float16,fp8,0,1.0146506627400715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,2,128,1,float16,float16,0,0.7721386750539144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,32,128,1,fp8,fp8,0,0.9299626350402832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,2,128,1,float16,fp8,0,0.7730933030446371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,2,128,1,fp8,fp8,0,0.7485013008117676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,4,128,1,float16,float16,0,0.7819999853769938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,4,128,1,float16,fp8,0,0.7868959903717041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,4,128,1,fp8,fp8,0,0.7661973635355631
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,8,128,1,fp8,fp8,0,0.7991680304209391
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,8,128,1,float16,float16,0,0.8049279848734537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,32,8,128,1,float16,fp8,0,0.8055040041605631
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,1,128,1,float16,float16,0,0.39312533537546795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,32,128,1,float16,float16,0,0.5350240071614584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,1,128,1,float16,fp8,0,0.3922186692555745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,32,128,1,float16,fp8,0,0.5162933270136515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,2,128,1,float16,float16,0,0.3978506724039714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,2,128,1,float16,fp8,0,0.3991626501083374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,1,128,1,fp8,fp8,0,0.364464004834493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,32,128,1,fp8,fp8,0,0.4737120072046916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,2,128,1,fp8,fp8,0,0.37993065516153973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,4,128,1,float16,float16,0,0.40143998463948566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,4,128,1,float16,fp8,0,0.40381332238515216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,8,128,1,float16,float16,0,0.41552531719207764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,4,128,1,fp8,fp8,0,0.3848693370819092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,8,128,1,fp8,fp8,0,0.40915199120839435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,32,8,128,1,float16,fp8,0,0.4132106701533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,32,128,1,float16,float16,0,0.2796906630198161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,1,128,1,float16,float16,0,0.20854934056599936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,1,128,1,fp8,fp8,0,0.19550933440526327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,1,128,1,float16,fp8,0,0.20895467201868692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,32,128,1,float16,fp8,0,0.26787734031677246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,2,128,1,float16,float16,0,0.2105706731478373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,32,128,1,fp8,fp8,0,0.24320000410079956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,2,128,1,float16,fp8,0,0.21158399184544882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,2,128,1,fp8,fp8,0,0.20106132825215658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,4,128,1,float16,float16,0,0.21341866254806519
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,4,128,1,fp8,fp8,0,0.2013333241144816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,4,128,1,float16,fp8,0,0.21357866128285727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,8,128,1,float16,fp8,0,0.21964265902837118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,8,128,1,float16,float16,0,0.22013866901397705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,32,8,128,1,fp8,fp8,0,0.21403199434280396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,32,128,1,float16,fp8,0,0.14632532993952432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,32,128,1,float16,float16,0,0.15131733814875284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,32,128,1,fp8,fp8,0,0.13143466909726462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,1,128,1,float16,fp8,0,0.1148426632086436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,1,128,1,float16,float16,0,0.1143839955329895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,1,128,1,fp8,fp8,0,0.10690133770306905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,2,128,1,float16,float16,0,0.11587199568748474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,2,128,1,float16,fp8,0,0.11613866686820984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,2,128,1,fp8,fp8,0,0.11036800344785054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,4,128,1,float16,float16,0,0.1179200013478597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,4,128,1,float16,fp8,0,0.11805867155392964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,4,128,1,fp8,fp8,0,0.11089600125948589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,8,128,1,float16,float16,0,0.12169599533081055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,8,128,1,float16,fp8,0,0.12134933471679688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,32,8,128,1,fp8,fp8,0,0.11427733302116394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,32,128,1,float16,fp8,0,0.08521599570910136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,32,128,1,float16,float16,0,0.08917333682378133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,32,128,1,fp8,fp8,0,0.07455466687679291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,1,128,1,fp8,fp8,0,0.059989333152770996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,2,128,1,float16,float16,0,0.06713066498438518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,1,128,1,float16,float16,0,0.06507733464241028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,2,128,1,float16,fp8,0,0.06632000207901001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,1,128,1,float16,fp8,0,0.0660693347454071
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,2,128,1,fp8,fp8,0,0.061333333452542625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,4,128,1,fp8,fp8,0,0.06243733565012614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,4,128,1,float16,fp8,0,0.06756266454855601
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,8,128,1,float16,fp8,0,0.06896533568700154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,4,128,1,float16,float16,0,0.06738133231798808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,8,128,1,float16,float16,0,0.06897599995136261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,32,128,1,float16,float16,0,0.047839999198913574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,32,8,128,1,fp8,fp8,0,0.06547733147939046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,1,128,1,float16,float16,0,0.03982399900754293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,32,128,1,float16,fp8,0,0.04724800089995066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,32,128,1,fp8,fp8,0,0.04418133199214935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,2,128,1,float16,float16,0,0.04005866746107737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,1,128,1,float16,fp8,0,0.040405333042144775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,1,128,1,fp8,fp8,0,0.0355679988861084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,2,128,1,fp8,fp8,0,0.036090667049090065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,4,128,1,float16,float16,0,0.04068800061941147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,4,128,1,float16,fp8,0,0.03996799886226654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,2,128,1,float16,fp8,0,0.04043733328580856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,4,128,1,fp8,fp8,0,0.03606933355331421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,8,128,1,float16,float16,0,0.041109333435694374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,8,128,1,fp8,fp8,0,0.038245332737763725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,32,8,128,1,float16,fp8,0,0.041050667564074196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,32,128,1,float16,float16,0,0.03139200061559677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,32,128,1,float16,fp8,0,0.030613332986831665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,1,128,1,float16,float16,0,0.027984000742435455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,1,128,1,fp8,fp8,0,0.026496000587940216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,1,128,1,float16,fp8,0,0.028517333169778187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,32,128,1,fp8,fp8,0,0.028421332438786823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,2,128,1,float16,fp8,0,0.029018667836983997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,2,128,1,fp8,fp8,0,0.026528000831604004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,2,128,1,float16,float16,0,0.02852799991766612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,4,128,1,float16,float16,0,0.028602667152881622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,4,128,1,float16,fp8,0,0.02852799991766612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,8,128,1,float16,fp8,0,0.02942399928967158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,8,128,1,fp8,fp8,0,0.02802666773398717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,4,128,1,fp8,fp8,0,0.02741866558790207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,32,8,128,1,float16,float16,0,0.02914133419593175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,1,128,1,float16,fp8,0,1.2842346827189128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,1,128,1,fp8,fp8,0,1.2469759782155354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,1,128,1,float16,float16,0,1.2824959754943848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,2,128,1,float16,float16,0,1.3141653537750244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,2,128,1,float16,fp8,0,1.3109172979990642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,2,128,1,fp8,fp8,0,1.280453364054362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,4,128,1,float16,float16,0,1.3331092993418376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,4,128,1,float16,fp8,0,1.3563146591186523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,4,128,1,fp8,fp8,0,1.3351252873738606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,8,128,1,float16,fp8,0,1.402234713236491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,8,128,1,float16,float16,0,1.4114665985107422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,32,8,128,1,fp8,fp8,0,1.4142932891845703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,32,128,1,float16,float16,0,0.9733066558837891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,32,128,1,float16,fp8,0,0.9309439659118652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,1,128,1,float16,float16,0,0.6528799931208292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,1,128,1,float16,fp8,0,0.6549706856409708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,1,128,1,fp8,fp8,0,0.6214239994684855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,2,128,1,float16,float16,0,0.6640853484471639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,2,128,1,float16,fp8,0,0.6658453146616617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,2,128,1,fp8,fp8,0,0.6476853291193644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,4,128,1,float16,float16,0,0.6754986445109049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,32,128,1,fp8,fp8,0,0.8428373336791992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,4,128,1,float16,fp8,0,0.6782080332438151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,8,128,1,float16,float16,0,0.6984960238138834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,4,128,1,fp8,fp8,0,0.6750186284383138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,8,128,1,float16,fp8,0,0.6932960351308187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,32,8,128,1,fp8,fp8,0,0.7078879674275717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,32,128,1,float16,float16,0,0.49186134338378906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,32,128,1,float16,fp8,0,0.46648534138997394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,1,128,1,float16,float16,0,0.33823466300964355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,1,128,1,float16,fp8,0,0.338368018468221
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,1,128,1,fp8,fp8,0,0.31943466265996295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,32,128,1,fp8,fp8,0,0.4318079948425293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,2,128,1,fp8,fp8,0,0.3356320063273112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,2,128,1,float16,fp8,0,0.34351468086242676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,2,128,1,float16,float16,0,0.34200533231099445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,4,128,1,float16,float16,0,0.3479573329289754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,4,128,1,float16,fp8,0,0.34798399607340497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,4,128,1,fp8,fp8,0,0.34405867258707684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,8,128,1,float16,float16,0,0.3616906801859538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,8,128,1,float16,fp8,0,0.35873599847157794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,32,8,128,1,fp8,fp8,0,0.3656426668167114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,32,128,1,float16,float16,0,0.2577600081761678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,1,128,1,float16,float16,0,0.18026133378346762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,32,128,1,float16,fp8,0,0.24615466594696045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,1,128,1,fp8,fp8,0,0.17241599162419638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,2,128,1,float16,float16,0,0.18138132492701212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,32,128,1,fp8,fp8,0,0.22235200802485147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,1,128,1,float16,fp8,0,0.18076266845067343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,2,128,1,float16,fp8,0,0.18223466475804648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,2,128,1,fp8,fp8,0,0.1787573297818502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,4,128,1,float16,float16,0,0.18572799364725748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,4,128,1,float16,fp8,0,0.18525866667429605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,4,128,1,fp8,fp8,0,0.17944000164667764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,8,128,1,float16,fp8,0,0.19121599197387695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,8,128,1,float16,float16,0,0.19252800941467285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,32,8,128,1,fp8,fp8,0,0.19129067659378052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,32,128,1,float16,float16,0,0.14200533429781595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,1,128,1,float16,float16,0,0.09934399525324504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,32,128,1,float16,fp8,0,0.13639466961224875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,32,128,1,fp8,fp8,0,0.12229333321253459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,1,128,1,float16,fp8,0,0.10036266843477885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,2,128,1,float16,float16,0,0.10130133231480916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,1,128,1,fp8,fp8,0,0.09693866968154907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,2,128,1,float16,fp8,0,0.10082667072614034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,2,128,1,fp8,fp8,0,0.09922132889429729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,4,128,1,float16,float16,0,0.10244799653689067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,4,128,1,float16,fp8,0,0.10251733660697937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,32,128,1,float16,float16,0,0.08714133501052856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,4,128,1,fp8,fp8,0,0.10128532846768697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,8,128,1,fp8,fp8,0,0.10427733262379964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,8,128,1,float16,fp8,0,0.10558933019638062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,32,128,1,float16,fp8,0,0.08362666765848796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,32,8,128,1,float16,float16,0,0.1067039966583252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,32,128,1,fp8,fp8,0,0.07182933390140533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,1,128,1,float16,float16,0,0.0590826670328776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,1,128,1,float16,fp8,0,0.05949333310127258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,1,128,1,fp8,fp8,0,0.054666668176651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,2,128,1,fp8,fp8,0,0.05542933444182078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,2,128,1,float16,fp8,0,0.05968533456325531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,4,128,1,float16,float16,0,0.060496002435684204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,2,128,1,float16,float16,0,0.05986666679382324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,4,128,1,float16,fp8,0,0.0602400004863739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,4,128,1,fp8,fp8,0,0.05726933479309082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,8,128,1,float16,float16,0,0.06250133117039998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,8,128,1,float16,fp8,0,0.0625600020090739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,32,8,128,1,fp8,fp8,0,0.06033066908518473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,32,128,1,float16,float16,0,0.04432533184687296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,32,128,1,fp8,fp8,0,0.04167466859022776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,1,128,1,float16,float16,0,0.03728533287843069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,32,128,1,float16,fp8,0,0.04364266494909922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,1,128,1,float16,fp8,0,0.0372533326347669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,1,128,1,fp8,fp8,0,0.03321066747109095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,2,128,1,float16,float16,0,0.03694933404525121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,2,128,1,float16,fp8,0,0.03783999880154928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,2,128,1,fp8,fp8,0,0.03317866722742716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,4,128,1,float16,float16,0,0.0378506655494372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,4,128,1,float16,fp8,0,0.03717333326737086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,4,128,1,fp8,fp8,0,0.03359466542800268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,8,128,1,float16,float16,0,0.03788800040880839
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,8,128,1,float16,fp8,0,0.038346665600935616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,32,8,128,1,fp8,fp8,0,0.03489066660404205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,32,128,1,float16,float16,0,0.02741333345572154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,32,128,1,float16,fp8,0,0.0283146674434344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,1,128,1,float16,float16,0,0.025263999899228413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,32,128,1,fp8,fp8,0,0.02510400116443634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,1,128,1,float16,fp8,0,0.025829332570234936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,1,128,1,fp8,fp8,0,0.02346133440732956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,2,128,1,float16,float16,0,0.025834667185942333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,2,128,1,float16,fp8,0,0.025589334468046825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,2,128,1,fp8,fp8,0,0.023733332753181458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,4,128,1,float16,float16,0,0.026186667382717133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,4,128,1,fp8,fp8,0,0.024127999941507976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,8,128,1,float16,float16,0,0.026261332134405773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,4,128,1,float16,fp8,0,0.0264533335963885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,8,128,1,float16,fp8,0,0.026714667677879333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,32,8,128,1,fp8,fp8,0,0.024490666886170704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,32,128,1,float16,float16,0,0.023567999402681988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,32,128,1,fp8,fp8,0,0.021594665944576263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,32,128,1,float16,fp8,0,0.02312533309062322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,1,128,1,float16,float16,0,0.022287999590237934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,1,128,1,fp8,fp8,0,0.020549333343903225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,2,128,1,float16,float16,0,0.02186133215824763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,1,128,1,float16,fp8,0,0.021898667017618816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,2,128,1,fp8,fp8,0,0.020970667401949566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,2,128,1,float16,fp8,0,0.022597332795461018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,4,128,1,float16,float16,0,0.02239999920129776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,4,128,1,float16,fp8,0,0.02231466770172119
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,8,128,1,float16,float16,0,0.022250667214393616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,4,128,1,fp8,fp8,0,0.02082666630546252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,8,128,1,float16,fp8,0,0.022543999056021374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,32,8,128,1,fp8,fp8,0,0.021727999051411945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,1,128,1,float16,float16,0,0.6018186807632446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,1,128,1,float16,fp8,0,0.6013866662979126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,1,128,1,fp8,fp8,0,0.6144853432973226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,2,128,1,float16,float16,0,0.6118186712265015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,2,128,1,fp8,fp8,0,0.645472009976705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,2,128,1,float16,fp8,0,0.6142773230870565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,4,128,1,float16,float16,0,0.6249386469523112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,4,128,1,float16,fp8,0,0.6232266823450724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,4,128,1,fp8,fp8,0,0.6701920032501221
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,8,128,1,float16,float16,0,0.647706667582194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,8,128,1,float16,fp8,0,0.6433546543121338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,32,128,1,float16,float16,0,0.5108799934387207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,32,8,128,1,fp8,fp8,0,0.7082133293151855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,32,128,1,float16,fp8,0,0.48843733469645184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,1,128,1,float16,float16,0,0.3141813278198242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,1,128,1,float16,fp8,0,0.3145493268966675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,2,128,1,float16,float16,0,0.31994666655858356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,1,128,1,fp8,fp8,0,0.31850665807724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,2,128,1,fp8,fp8,0,0.32808534304300946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,2,128,1,float16,fp8,0,0.3195359905560811
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,32,128,1,fp8,fp8,0,0.4329013427098592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,4,128,1,float16,float16,0,0.3245546619097392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,4,128,1,float16,fp8,0,0.3243680000305176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,4,128,1,fp8,fp8,0,0.3417919874191284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,8,128,1,float16,float16,0,0.3376266558965047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,8,128,1,float16,fp8,0,0.33483731746673584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,1,128,1,float16,float16,0,0.17002665996551514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,32,128,1,float16,float16,0,0.2680533329645793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,32,8,128,1,fp8,fp8,0,0.3632426659266154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,32,128,1,float16,fp8,0,0.2606026728947957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,1,128,1,float16,fp8,0,0.16963734229405722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,1,128,1,fp8,fp8,0,0.17189866304397583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,32,128,1,fp8,fp8,0,0.22427199284235635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,2,128,1,float16,float16,0,0.17148266235987344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,2,128,1,float16,fp8,0,0.17322667439778647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,4,128,1,float16,fp8,0,0.17483733097712198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,4,128,1,float16,float16,0,0.1759893298149109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,2,128,1,fp8,fp8,0,0.17826133966445923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,4,128,1,fp8,fp8,0,0.18077333768208823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,8,128,1,float16,float16,0,0.1835413376490275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,1,128,1,float16,float16,0,0.0957493285338084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,8,128,1,float16,fp8,0,0.18182400862375894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,32,8,128,1,fp8,fp8,0,0.19202667474746704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,32,128,1,float16,fp8,0,0.14309333761533102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,1,128,1,float16,fp8,0,0.09533333778381348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,32,128,1,float16,float16,0,0.14615466197331747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,32,128,1,fp8,fp8,0,0.12292800347010295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,1,128,1,fp8,fp8,0,0.0962559978167216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,2,128,1,float16,float16,0,0.09667199850082397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,2,128,1,float16,fp8,0,0.09726400176684062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,2,128,1,fp8,fp8,0,0.09744532903035481
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,4,128,1,float16,float16,0,0.09884799520174663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,4,128,1,float16,fp8,0,0.0990559955437978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,8,128,1,float16,float16,0,0.10409067074457805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,8,128,1,float16,fp8,0,0.10292266805966695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,4,128,1,fp8,fp8,0,0.09957333405812581
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,32,8,128,1,fp8,fp8,0,0.1046453317006429
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,32,128,1,float16,float16,0,0.08483733733495076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,32,128,1,float16,fp8,0,0.08043733239173889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,32,128,1,fp8,fp8,0,0.07080533107121785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,1,128,1,float16,float16,0,0.057429333527882896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,1,128,1,float16,fp8,0,0.05675200124581655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,2,128,1,float16,float16,0,0.057392001152038574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,2,128,1,float16,fp8,0,0.05721066892147064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,2,128,1,fp8,fp8,0,0.05654400090376536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,4,128,1,float16,float16,0,0.057301332553227745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,1,128,1,fp8,fp8,0,0.054458667834599815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,4,128,1,float16,fp8,0,0.058778668443361916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,4,128,1,fp8,fp8,0,0.05635733405749003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,8,128,1,float16,fp8,0,0.06007466713587443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,8,128,1,float16,float16,0,0.06016000111897787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,32,8,128,1,fp8,fp8,0,0.059994667768478394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,32,128,1,float16,float16,0,0.04275199770927429
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,1,128,1,float16,float16,0,0.03541333228349686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,32,128,1,float16,fp8,0,0.04163199911514918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,1,128,1,float16,fp8,0,0.03619199991226196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,1,128,1,fp8,fp8,0,0.03310933212439219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,32,128,1,fp8,fp8,0,0.041146665811538696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,2,128,1,float16,float16,0,0.03602666656176249
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,2,128,1,float16,fp8,0,0.035887998839219414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,2,128,1,fp8,fp8,0,0.03348266581694285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,4,128,1,float16,fp8,0,0.03641066700220108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,4,128,1,fp8,fp8,0,0.03363733241955439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,8,128,1,float16,float16,0,0.03655466685692469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,8,128,1,float16,fp8,0,0.037050666908423104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,4,128,1,float16,float16,0,0.03623999903599421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,32,8,128,1,fp8,fp8,0,0.0351200004418691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,32,128,1,float16,float16,0,0.02685333291689555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,32,128,1,float16,fp8,0,0.026890667776266735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,1,128,1,float16,fp8,0,0.024490666886170704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,1,128,1,float16,float16,0,0.02481599897146225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,32,128,1,fp8,fp8,0,0.02550933261712392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,2,128,1,float16,float16,0,0.024688000480333965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,2,128,1,float16,fp8,0,0.025290665527184803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,1,128,1,fp8,fp8,0,0.02314666658639908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,2,128,1,fp8,fp8,0,0.023189333577950794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,4,128,1,float16,float16,0,0.025055999557177227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,4,128,1,float16,fp8,0,0.025536000728607178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,4,128,1,fp8,fp8,0,0.024373332659403484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,8,128,1,float16,float16,0,0.025413334369659424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,8,128,1,float16,fp8,0,0.02588266630967458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,32,128,1,float16,float16,0,0.021231998999913532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,32,128,1,float16,fp8,0,0.022085333863894146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,32,128,1,fp8,fp8,0,0.021722666919231415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,1,128,1,float16,fp8,0,0.020762667059898376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,1,128,1,float16,float16,0,0.020879998803138733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,32,8,128,1,fp8,fp8,0,0.024656000236670177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,2,128,1,float16,float16,0,0.02075733368595441
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,1,128,1,fp8,fp8,0,0.02041600023706754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,2,128,1,float16,fp8,0,0.02103466788927714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,2,128,1,fp8,fp8,0,0.02081599955757459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,4,128,1,float16,float16,0,0.020960000654061634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,4,128,1,float16,fp8,0,0.021583999196688335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,8,128,1,float16,float16,0,0.021322667598724365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,8,128,1,fp8,fp8,0,0.021520001192887623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,4,128,1,fp8,fp8,0,0.021087999145189922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,32,8,128,1,float16,fp8,0,0.021338666478792827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,32,128,1,float16,float16,0,0.019914666811625164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,32,128,1,float16,fp8,0,0.01951466624935468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,32,128,1,fp8,fp8,0,0.019893333315849304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,1,128,1,float16,float16,0,0.01956266661485036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,1,128,1,float16,fp8,0,0.020090666910012562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,2,128,1,float16,fp8,0,0.020181333025296528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,2,128,1,float16,float16,0,0.019717333217461903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,2,128,1,fp8,fp8,0,0.01993600030740102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,4,128,1,float16,float16,0,0.020021333048741024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,1,128,1,fp8,fp8,0,0.01897066707412402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,4,128,1,float16,fp8,0,0.020138667275508244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,4,128,1,fp8,fp8,0,0.019802667200565338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,8,128,1,float16,float16,0,0.019920000185569126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,8,128,1,float16,fp8,0,0.020154666155576706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,32,8,128,1,fp8,fp8,0,0.019658666104078293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,1,128,1,float16,float16,0,0.3901013135910034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,1,128,1,fp8,fp8,0,0.420144001642863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,2,128,1,float16,float16,0,0.3946933348973592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,2,128,1,float16,fp8,0,0.39446401596069336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,2,128,1,fp8,fp8,0,0.44172267119089764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,1,128,1,float16,fp8,0,0.3893760045369466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,4,128,1,float16,float16,0,0.3983786503473918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,4,128,1,fp8,fp8,0,0.4503946701685588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,4,128,1,float16,fp8,0,0.39930665493011475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,8,128,1,float16,float16,0,0.4119253158569336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,8,128,1,fp8,fp8,0,0.47577067216237384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,32,8,128,1,float16,fp8,0,0.4089706738789876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,32,128,1,float16,fp8,0,0.25413866837819415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,32,128,1,float16,float16,0,0.2656480073928833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,1,128,1,float16,float16,0,0.20442666610081991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,1,128,1,fp8,fp8,0,0.2226933240890503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,1,128,1,float16,fp8,0,0.20467199881871542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,2,128,1,float16,float16,0,0.20684800545374551
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,2,128,1,float16,fp8,0,0.2072746753692627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,32,128,1,fp8,fp8,0,0.2714560031890869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,2,128,1,fp8,fp8,0,0.2294506629308065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,4,128,1,float16,float16,0,0.209824005762736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,4,128,1,float16,fp8,0,0.20805333058039346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,4,128,1,fp8,fp8,0,0.23253333568572998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,8,128,1,float16,float16,0,0.21811733643213907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,8,128,1,fp8,fp8,0,0.2436586618423462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,32,8,128,1,float16,fp8,0,0.21549866596857706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,32,128,1,float16,fp8,0,0.13860799868901572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,1,128,1,float16,float16,0,0.11070400476455688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,32,128,1,float16,float16,0,0.14430933197339377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,1,128,1,float16,fp8,0,0.11173333724339803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,2,128,1,float16,float16,0,0.11124266187349956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,1,128,1,fp8,fp8,0,0.12265599767367046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,32,128,1,fp8,fp8,0,0.1453386644522349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,2,128,1,float16,fp8,0,0.11177066961924235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,2,128,1,fp8,fp8,0,0.12548800309499106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,4,128,1,fp8,fp8,0,0.12570133805274963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,4,128,1,float16,float16,0,0.11373866597811381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,4,128,1,float16,fp8,0,0.11412800351778667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,8,128,1,float16,float16,0,0.11860266327857971
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,8,128,1,float16,fp8,0,0.11756799618403117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,32,8,128,1,fp8,fp8,0,0.1304480036099752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,32,128,1,float16,float16,0,0.08163199822107951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,32,128,1,float16,fp8,0,0.07885866860548656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,32,128,1,fp8,fp8,0,0.08366400003433228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,1,128,1,float16,fp8,0,0.06326933205127716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,1,128,1,float16,float16,0,0.06320533156394958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,1,128,1,fp8,fp8,0,0.06854400038719177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,2,128,1,float16,float16,0,0.06397866706053416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,2,128,1,float16,fp8,0,0.06353066861629486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,2,128,1,fp8,fp8,0,0.06965333223342896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,4,128,1,float16,float16,0,0.06435733536879222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,4,128,1,float16,fp8,0,0.06369066735108693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,4,128,1,fp8,fp8,0,0.07048533360163371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,8,128,1,float16,float16,0,0.066170667608579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,8,128,1,float16,fp8,0,0.06625066697597504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,32,8,128,1,fp8,fp8,0,0.07433600227038066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,32,128,1,float16,float16,0,0.04304533203442892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,1,128,1,float16,float16,0,0.03875199953715006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,1,128,1,float16,fp8,0,0.03860799968242645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,1,128,1,fp8,fp8,0,0.04116799930731455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,32,128,1,float16,fp8,0,0.041850666205088295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,32,128,1,fp8,fp8,0,0.04909333089987437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,2,128,1,float16,fp8,0,0.0388373335202535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,2,128,1,float16,float16,0,0.03862400104602178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,2,128,1,fp8,fp8,0,0.040752001106739044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,4,128,1,float16,float16,0,0.038618666430314384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,8,128,1,float16,float16,0,0.039408000806967415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,4,128,1,float16,fp8,0,0.038880000511805214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,8,128,1,fp8,fp8,0,0.043231998880704246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,8,128,1,float16,fp8,0,0.03958400090535482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,32,4,128,1,fp8,fp8,0,0.041082667807737984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,32,128,1,float16,float16,0,0.027717334528764088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,1,128,1,float16,fp8,0,0.026144000391165417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,1,128,1,fp8,fp8,0,0.027061333258946735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,1,128,1,float16,float16,0,0.026357332865397137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,32,128,1,fp8,fp8,0,0.029418667157491047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,32,128,1,float16,fp8,0,0.027786667148272198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,2,128,1,float16,fp8,0,0.026575999955336254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,2,128,1,fp8,fp8,0,0.027104000250498455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,2,128,1,float16,float16,0,0.025941332181294758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,4,128,1,float16,float16,0,0.026543999711672466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,4,128,1,fp8,fp8,0,0.028234665592511494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,4,128,1,float16,fp8,0,0.02718399961789449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,8,128,1,float16,float16,0,0.0268053337931633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,8,128,1,float16,fp8,0,0.027242665489514668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,32,8,128,1,fp8,fp8,0,0.02863466739654541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,32,128,1,float16,fp8,0,0.021530665457248688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,32,128,1,fp8,fp8,0,0.022074667116006214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,1,128,1,float16,fp8,0,0.019695999721686046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,1,128,1,float16,float16,0,0.02037866661945979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,32,128,1,float16,float16,0,0.021381333470344543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,1,128,1,fp8,fp8,0,0.020773333807786305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,2,128,1,float16,float16,0,0.020224000016848247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,2,128,1,fp8,fp8,0,0.02091199904680252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,4,128,1,float16,float16,0,0.02012266715367635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,2,128,1,float16,fp8,0,0.020165332903464634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,4,128,1,float16,fp8,0,0.020586666961510975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,8,128,1,float16,float16,0,0.020554666717847187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,8,128,1,float16,fp8,0,0.020597333709398907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,4,128,1,fp8,fp8,0,0.02083733429511388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,32,128,1,float16,float16,0,0.018239999810854595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,32,8,128,1,fp8,fp8,0,0.021370666722456615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,1,128,1,float16,float16,0,0.01844800015290578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,32,128,1,float16,fp8,0,0.0186666672428449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,32,128,1,fp8,fp8,0,0.020181333025296528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,1,128,1,float16,fp8,0,0.01887999971707662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,2,128,1,float16,float16,0,0.018266666680574417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,2,128,1,fp8,fp8,0,0.019765333582957584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,1,128,1,fp8,fp8,0,0.019280000279347103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,2,128,1,float16,fp8,0,0.018965333700180054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,4,128,1,fp8,fp8,0,0.019968000551064808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,4,128,1,float16,float16,0,0.018394666413466137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,8,128,1,float16,fp8,0,0.019002666076024372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,8,128,1,float16,float16,0,0.018613333503405254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,4,128,1,float16,fp8,0,0.019029332945744198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,32,128,1,float16,float16,0,0.017765333255132038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,32,8,128,1,fp8,fp8,0,0.019626667102177937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,32,128,1,float16,fp8,0,0.018383999665578205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,1,128,1,float16,float16,0,0.017792000124851864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,32,128,1,fp8,fp8,0,0.018901333212852478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,1,128,1,float16,fp8,0,0.018288000176350277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,2,128,1,float16,float16,0,0.017887999614079792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,1,128,1,fp8,fp8,0,0.019199999670187633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,2,128,1,float16,fp8,0,0.0183999997874101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,4,128,1,float16,float16,0,0.017711999515692394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,2,128,1,fp8,fp8,0,0.018725333114465077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,4,128,1,float16,fp8,0,0.018538666268189747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,4,128,1,fp8,fp8,0,0.018906666586796444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,8,128,1,float16,float16,0,0.017781333376963932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,8,128,1,float16,fp8,0,0.01834133391578992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,32,8,128,1,fp8,fp8,0,0.019093333433071773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,1,128,1,float16,fp8,0,0.2696640094121297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,1,128,1,fp8,fp8,0,0.3410773277282715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,2,128,1,float16,float16,0,0.27591466903686523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,1,128,1,float16,float16,0,0.2697333296140035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,2,128,1,float16,fp8,0,0.2746506730715434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,2,128,1,fp8,fp8,0,0.35181868076324463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,4,128,1,float16,float16,0,0.2816480000813802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,4,128,1,float16,fp8,0,0.2810720006624858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,8,128,1,float16,float16,0,0.29101334015528363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,4,128,1,fp8,fp8,0,0.35170666376749676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,8,128,1,fp8,fp8,0,0.36243200302124023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,32,8,128,1,float16,fp8,0,0.2901386618614197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,32,128,1,float16,float16,0,0.17509865760803223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,1,128,1,float16,float16,0,0.1430293321609497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,1,128,1,float16,fp8,0,0.14308266838391623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,32,128,1,float16,fp8,0,0.17126933733622232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,1,128,1,fp8,fp8,0,0.18136000633239746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,32,128,1,fp8,fp8,0,0.20549333095550537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,2,128,1,float16,float16,0,0.1451520025730133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,2,128,1,float16,fp8,0,0.14553067088127136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,2,128,1,fp8,fp8,0,0.18432533740997314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,4,128,1,float16,float16,0,0.1485919952392578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,4,128,1,fp8,fp8,0,0.18424532810846964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,8,128,1,float16,float16,0,0.15475733081499735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,8,128,1,float16,fp8,0,0.15355199575424194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,4,128,1,float16,fp8,0,0.14851199587186178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,32,8,128,1,fp8,fp8,0,0.18954666455586752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,32,128,1,float16,float16,0,0.09804266691207886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,32,128,1,float16,fp8,0,0.09506666660308838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,1,128,1,float16,float16,0,0.07839466631412506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,2,128,1,float16,float16,0,0.07828266421953838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,1,128,1,fp8,fp8,0,0.10006399949391682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,32,128,1,fp8,fp8,0,0.11340799927711487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,1,128,1,float16,fp8,0,0.07774400214354198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,2,128,1,fp8,fp8,0,0.10047466556231181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,2,128,1,float16,fp8,0,0.07784000039100647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,4,128,1,float16,fp8,0,0.07910933097203572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,4,128,1,float16,float16,0,0.07902400195598602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,4,128,1,fp8,fp8,0,0.10262933373451233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,8,128,1,float16,float16,0,0.08146666487058003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,32,128,1,float16,float16,0,0.05037866532802582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,32,128,1,float16,fp8,0,0.049471999208132424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,8,128,1,float16,fp8,0,0.08155733346939087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,32,8,128,1,fp8,fp8,0,0.10486400127410889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,32,128,1,fp8,fp8,0,0.06492266555627187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,1,128,1,float16,float16,0,0.04539200166861216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,1,128,1,float16,fp8,0,0.04571733375390371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,1,128,1,fp8,fp8,0,0.05649066468079885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,2,128,1,float16,float16,0,0.045968001087506614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,2,128,1,fp8,fp8,0,0.05641066531340281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,2,128,1,float16,fp8,0,0.046165332198143005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,4,128,1,float16,float16,0,0.04609066744645437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,4,128,1,fp8,fp8,0,0.057434668143590294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,8,128,1,float16,fp8,0,0.046623999873797096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,4,128,1,float16,fp8,0,0.04571733375390371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,8,128,1,float16,float16,0,0.046367997924486794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,32,8,128,1,fp8,fp8,0,0.05842133363087972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,32,128,1,float16,float16,0,0.030746666093667347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,32,128,1,fp8,fp8,0,0.03733866661787033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,32,128,1,float16,fp8,0,0.030634666482607525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,1,128,1,float16,float16,0,0.02980799973011017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,1,128,1,fp8,fp8,0,0.03557866563399633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,2,128,1,float16,float16,0,0.030063999195893604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,1,128,1,float16,fp8,0,0.03014400104681651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,2,128,1,fp8,fp8,0,0.035717333356539406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,4,128,1,float16,float16,0,0.030095999439557392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,4,128,1,float16,fp8,0,0.03072533259789149
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,4,128,1,fp8,fp8,0,0.036389333506425224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,8,128,1,float16,float16,0,0.030282666285832722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,2,128,1,float16,fp8,0,0.02994133283694585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,8,128,1,fp8,fp8,0,0.03700266778469086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,32,128,1,float16,fp8,0,0.02254933367172877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,32,128,1,float16,float16,0,0.022277332842350006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,32,8,128,1,float16,fp8,0,0.031162666777769726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,1,128,1,float16,float16,0,0.021514666577180225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,32,128,1,fp8,fp8,0,0.025781333446502686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,2,128,1,float16,float16,0,0.021989333132902782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,1,128,1,float16,fp8,0,0.021967999637126923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,4,128,1,float16,float16,0,0.022128000855445862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,1,128,1,fp8,fp8,0,0.02477866659561793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,2,128,1,float16,fp8,0,0.021898667017618816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,4,128,1,float16,fp8,0,0.02205866575241089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,2,128,1,fp8,fp8,0,0.024714666108290356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,4,128,1,fp8,fp8,0,0.025568000972270966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,8,128,1,float16,fp8,0,0.022367998957633972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,8,128,1,fp8,fp8,0,0.025418666501839954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,32,128,1,float16,float16,0,0.018432000031073887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,32,8,128,1,float16,float16,0,0.021909333765506744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,32,128,1,float16,fp8,0,0.018464000274737675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,1,128,1,float16,float16,0,0.01815466706951459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,32,128,1,fp8,fp8,0,0.020037333170572918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,1,128,1,float16,fp8,0,0.017909333109855652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,2,128,1,float16,float16,0,0.017898666361967724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,1,128,1,fp8,fp8,0,0.0195573332409064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,2,128,1,fp8,fp8,0,0.019754666835069656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,2,128,1,float16,fp8,0,0.0184906671444575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,4,128,1,float16,float16,0,0.018112000077962875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,8,128,1,float16,float16,0,0.018357332795858383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,4,128,1,fp8,fp8,0,0.020058666666348774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,8,128,1,fp8,fp8,0,0.020143999407688778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,8,128,1,float16,fp8,0,0.018757333358128864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,32,128,1,float16,float16,0,0.017370666066805523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,32,4,128,1,float16,fp8,0,0.01801066721479098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,32,128,1,float16,fp8,0,0.017946666727463405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,32,128,1,fp8,fp8,0,0.019658666104078293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,1,128,1,float16,float16,0,0.01729600007335345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,1,128,1,float16,fp8,0,0.017551999539136887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,2,128,1,float16,float16,0,0.017477333545684814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,1,128,1,fp8,fp8,0,0.018976000448067982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,2,128,1,fp8,fp8,0,0.018922666708628338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,4,128,1,float16,float16,0,0.017477333545684814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,2,128,1,float16,fp8,0,0.017637333522240322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,4,128,1,fp8,fp8,0,0.018858666221300762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,4,128,1,float16,fp8,0,0.017946666727463405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,8,128,1,float16,float16,0,0.017664000391960144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,8,128,1,float16,fp8,0,0.01754133279124896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,32,8,128,1,fp8,fp8,0,0.01876266673207283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,32,128,1,float16,float16,0,0.01674666628241539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,1,128,1,float16,float16,0,0.01692266638080279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,32,128,1,fp8,fp8,0,0.01899733394384384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,1,128,1,float16,fp8,0,0.017610666652520496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,1,128,1,fp8,fp8,0,0.01844266677896182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,32,128,1,float16,fp8,0,0.01701333373785019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,2,128,1,float16,float16,0,0.01682666689157486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,2,128,1,float16,fp8,0,0.017045332739750545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,2,128,1,fp8,fp8,0,0.01814933369557063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,4,128,1,float16,fp8,0,0.01730666682124138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,4,128,1,float16,float16,0,0.0169813334941864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,4,128,1,fp8,fp8,0,0.01852799952030182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,8,128,1,fp8,fp8,0,0.01858666663368543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,8,128,1,float16,fp8,0,0.01739199956258138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,32,8,128,1,float16,float16,0,0.01706133286158244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,1,128,1,float16,float16,0,0.22748800118764242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,1,128,1,float16,fp8,0,0.2278346618016561
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,1,128,1,fp8,fp8,0,0.30214933554331463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,2,128,1,float16,float16,0,0.23075199127197266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,2,128,1,fp8,fp8,0,0.3054666717847188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,2,128,1,float16,fp8,0,0.23061867554982504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,4,128,1,float16,fp8,0,0.23268800973892212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,4,128,1,fp8,fp8,0,0.3049493432044983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,4,128,1,float16,float16,0,0.2337119976679484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,8,128,1,float16,float16,0,0.23944000403086343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,8,128,1,float16,fp8,0,0.23825067281723022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,32,8,128,1,fp8,fp8,0,0.30958400170008343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,32,128,1,float16,float16,0,0.1313920021057129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,1,128,1,float16,fp8,0,0.1216266651948293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,1,128,1,fp8,fp8,0,0.16012799739837646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,32,128,1,float16,fp8,0,0.12865599989891052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,32,128,1,fp8,fp8,0,0.1713226636250814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,2,128,1,float16,float16,0,0.12196800112724304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,1,128,1,float16,float16,0,0.12111999591191609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,2,128,1,fp8,fp8,0,0.16207999984423319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,4,128,1,float16,float16,0,0.12198932965596516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,4,128,1,float16,fp8,0,0.1223306655883789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,4,128,1,fp8,fp8,0,0.1628373364607493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,2,128,1,float16,fp8,0,0.12196800112724304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,8,128,1,float16,float16,0,0.12545067071914673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,8,128,1,float16,fp8,0,0.12486400206883748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,32,8,128,1,fp8,fp8,0,0.1662399967511495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,32,128,1,float16,float16,0,0.0697920024394989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,32,128,1,float16,fp8,0,0.0681333343187968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,32,128,1,fp8,fp8,0,0.09575999776522319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,1,128,1,float16,float16,0,0.06763199965159099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,1,128,1,float16,fp8,0,0.06735999882221222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,2,128,1,float16,float16,0,0.06692266464233398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,2,128,1,float16,fp8,0,0.0675786683956782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,2,128,1,fp8,fp8,0,0.0886240005493164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,1,128,1,fp8,fp8,0,0.08853333195050557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,4,128,1,float16,float16,0,0.06761600077152252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,4,128,1,float16,fp8,0,0.06785599887371063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,4,128,1,fp8,fp8,0,0.08930666248003642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,8,128,1,float16,float16,0,0.06778666873772939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,8,128,1,float16,fp8,0,0.06834133466084798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,32,8,128,1,fp8,fp8,0,0.08995200196901958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,32,128,1,float16,fp8,0,0.04029866556326548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,1,128,1,float16,float16,0,0.040965333580970764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,32,128,1,fp8,fp8,0,0.05300800005594889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,32,128,1,float16,float16,0,0.039701332648595176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,1,128,1,float16,fp8,0,0.04163199911514918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,1,128,1,fp8,fp8,0,0.05195199946562449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,2,128,1,float16,float16,0,0.04077333211898804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,2,128,1,fp8,fp8,0,0.051962668697039284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,2,128,1,float16,fp8,0,0.04211199780305227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,4,128,1,float16,float16,0,0.041509332756201424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,4,128,1,fp8,fp8,0,0.05249600112438202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,8,128,1,float16,float16,0,0.04142933338880539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,4,128,1,float16,fp8,0,0.04138666639725367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,8,128,1,fp8,fp8,0,0.05340266724427541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,32,8,128,1,float16,fp8,0,0.0413973331451416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,32,128,1,float16,float16,0,0.027093333502610523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,1,128,1,float16,float16,0,0.027493332823117573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,32,128,1,fp8,fp8,0,0.03329599897066752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,32,128,1,float16,fp8,0,0.0295413335164388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,1,128,1,fp8,fp8,0,0.033573334415753685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,2,128,1,float16,float16,0,0.02759466568628947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,1,128,1,float16,fp8,0,0.027834666272004444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,2,128,1,fp8,fp8,0,0.03291733314593633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,2,128,1,float16,fp8,0,0.02762666592995326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,4,128,1,float16,float16,0,0.027744000156720478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,4,128,1,fp8,fp8,0,0.0341386670867602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,8,128,1,float16,float16,0,0.028207999964555103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,4,128,1,float16,fp8,0,0.028021333118279774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,8,128,1,fp8,fp8,0,0.03439466655254364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,32,128,1,float16,float16,0,0.02041600023706754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,32,128,1,float16,fp8,0,0.020741333564122517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,32,8,128,1,float16,fp8,0,0.02784000088771184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,32,128,1,fp8,fp8,0,0.0244159996509552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,1,128,1,float16,float16,0,0.020261333634455998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,1,128,1,fp8,fp8,0,0.023797333240509033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,1,128,1,float16,fp8,0,0.020842666427294414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,2,128,1,float16,float16,0,0.020768000433842342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,2,128,1,float16,fp8,0,0.020992000897725422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,4,128,1,float16,float16,0,0.021146667500336964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,4,128,1,fp8,fp8,0,0.024112001061439514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,4,128,1,float16,fp8,0,0.021226666867733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,8,128,1,float16,fp8,0,0.021488000949223835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,8,128,1,float16,float16,0,0.020869334538777668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,2,128,1,fp8,fp8,0,0.023946667710940044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,32,8,128,1,fp8,fp8,0,0.024090667565663654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,32,128,1,float16,fp8,0,0.018298666924238205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,32,128,1,float16,float16,0,0.01788266624013583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,32,128,1,fp8,fp8,0,0.02006400004029274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,1,128,1,float16,float16,0,0.017498667041460674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,2,128,1,float16,float16,0,0.01721599946419398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,2,128,1,float16,fp8,0,0.017887999614079792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,2,128,1,fp8,fp8,0,0.01932799940307935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,4,128,1,float16,float16,0,0.017530667285124462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,1,128,1,float16,fp8,0,0.01766933376590411
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,4,128,1,float16,fp8,0,0.017957333475351334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,8,128,1,float16,float16,0,0.01766933376590411
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,4,128,1,fp8,fp8,0,0.019343999524911244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,1,128,1,fp8,fp8,0,0.019424000134070713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,8,128,1,float16,fp8,0,0.017808000246683758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,32,8,128,1,fp8,fp8,0,0.01942933350801468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,32,128,1,fp8,fp8,0,0.0189280000825723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,32,128,1,float16,fp8,0,0.01741333305835724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,1,128,1,float16,float16,0,0.017103999853134155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,32,128,1,float16,float16,0,0.01894933357834816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,1,128,1,fp8,fp8,0,0.01860800012946129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,2,128,1,float16,float16,0,0.016901332885026932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,1,128,1,float16,fp8,0,0.01728533332546552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,2,128,1,float16,fp8,0,0.01716800034046173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,2,128,1,fp8,fp8,0,0.019013332823912304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,4,128,1,float16,float16,0,0.016976000120242436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,4,128,1,float16,fp8,0,0.017674667139848072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,4,128,1,fp8,fp8,0,0.0185759998857975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,8,128,1,float16,float16,0,0.016645333419243496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,8,128,1,fp8,fp8,0,0.0184906671444575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,32,128,1,float16,float16,0,0.01635733370979627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,32,128,1,float16,fp8,0,0.016677333662907284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,32,8,128,1,float16,fp8,0,0.017535999417304993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,1,128,1,float16,float16,0,0.01703466723362605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,1,128,1,float16,fp8,0,0.016906666258970898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,32,128,1,fp8,fp8,0,0.018629333625237148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,1,128,1,fp8,fp8,0,0.01834133391578992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,2,128,1,float16,fp8,0,0.017114666601022083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,2,128,1,float16,float16,0,0.016607999801635742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,4,128,1,float16,float16,0,0.01699200024207433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,4,128,1,float16,fp8,0,0.01727466657757759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,4,128,1,fp8,fp8,0,0.01865600049495697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,8,128,1,float16,fp8,0,0.017210666090250015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,8,128,1,float16,float16,0,0.01685333376129468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,2,128,1,fp8,fp8,0,0.0185759998857975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,32,8,128,1,fp8,fp8,0,0.018565333137909572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,1,128,1,fp8,fp8,0,16.504032135009766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,2,128,1,fp8,fp8,0,16.405413309733074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,1,128,1,float16,fp8,0,25.607070922851562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,1,128,1,float16,float16,0,25.09764353434245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,2,128,1,float16,fp8,0,25.229771931966145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,2,128,1,float16,float16,0,25.422203063964844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,4,128,1,float16,float16,0,25.037333170572918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,4,128,1,float16,fp8,0,25.15630849202474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,4,128,1,fp8,fp8,0,16.40621821085612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,24,128,1,float16,float16,0,12.7148806254069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,8,128,1,fp8,fp8,0,16.586165110270183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,8,128,1,float16,fp8,0,25.34197235107422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,24,128,1,float16,fp8,0,12.486549377441406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,24,8,128,1,float16,float16,0,25.837498982747395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,1,128,1,float16,float16,0,12.61688486735026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,1,128,1,float16,fp8,0,12.557632446289062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,1,128,1,fp8,fp8,0,8.385018666585287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,2,128,1,float16,fp8,0,12.579861958821615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,2,128,1,float16,float16,0,12.665770212809244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,2,128,1,fp8,fp8,0,8.496821085611979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,4,128,1,float16,float16,0,12.573844909667969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,24,128,1,fp8,fp8,0,8.345130920410156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,4,128,1,float16,fp8,0,12.792560577392578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,4,128,1,fp8,fp8,0,8.242997487386068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,8,128,1,fp8,fp8,0,8.600298563639322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,8,128,1,float16,fp8,0,12.650479634602865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,24,8,128,1,float16,float16,0,12.61612319946289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,1,128,1,float16,float16,0,6.26141357421875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,24,128,1,float16,fp8,0,6.133679707845052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,1,128,1,float16,fp8,0,6.283429463704427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,24,128,1,float16,float16,0,6.169541041056315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,24,128,1,fp8,fp8,0,4.21285343170166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,2,128,1,float16,float16,0,6.164000193277995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,1,128,1,fp8,fp8,0,4.167445182800293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,2,128,1,float16,fp8,0,6.269589106241862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,2,128,1,fp8,fp8,0,4.115344047546387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,4,128,1,float16,float16,0,6.312042872111003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,4,128,1,fp8,fp8,0,4.132986704508464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,4,128,1,float16,fp8,0,6.069925308227539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,8,128,1,float16,float16,0,6.243792215983073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,8,128,1,fp8,fp8,0,4.1898454030354815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,24,8,128,1,float16,fp8,0,6.389935811360677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,1,128,1,float16,float16,0,3.128437360127767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,24,128,1,float16,float16,0,3.146911938985189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,24,128,1,float16,fp8,0,3.2057387034098306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,1,128,1,fp8,fp8,0,2.1748852729797363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,24,128,1,fp8,fp8,0,2.185962677001953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,1,128,1,float16,fp8,0,3.105205217997233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,2,128,1,float16,float16,0,3.1445067723592124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,2,128,1,fp8,fp8,0,2.175466696421305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,4,128,1,float16,float16,0,3.197861353556315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,2,128,1,float16,fp8,0,3.1069119771321616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,4,128,1,float16,fp8,0,3.155856132507324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,4,128,1,fp8,fp8,0,2.179807980855306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,8,128,1,float16,float16,0,3.1868906021118164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,8,128,1,float16,fp8,0,3.1439412434895835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,24,8,128,1,fp8,fp8,0,2.1942453384399414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,1,128,1,fp8,fp8,0,9.629552205403646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,1,128,1,float16,float16,0,14.777290344238281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,1,128,1,float16,fp8,0,14.427023569742838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,2,128,1,float16,fp8,0,14.305610656738281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,2,128,1,float16,float16,0,14.698219299316406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,2,128,1,fp8,fp8,0,9.704005559285482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,4,128,1,float16,float16,0,14.706671396891275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,4,128,1,fp8,fp8,0,9.95091183980306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,4,128,1,float16,fp8,0,14.633061726888021
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,8,128,1,float16,fp8,0,14.390703837076822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,8,128,1,fp8,fp8,0,9.791402816772461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,24,128,1,float16,float16,0,7.373215993245442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,24,8,128,1,float16,float16,0,14.610628763834635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,24,128,1,float16,fp8,0,7.233925501505534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,1,128,1,float16,float16,0,7.319162368774414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,1,128,1,float16,fp8,0,7.179557164510091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,1,128,1,fp8,fp8,0,4.768704096476237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,2,128,1,float16,float16,0,7.203653335571289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,2,128,1,float16,fp8,0,7.113040288289388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,24,128,1,fp8,fp8,0,5.064490636189778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,4,128,1,float16,float16,0,7.291274388631185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,2,128,1,fp8,fp8,0,4.815018653869629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,4,128,1,fp8,fp8,0,4.769258817036946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,4,128,1,float16,fp8,0,7.176933288574219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,8,128,1,float16,float16,0,7.26365852355957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,8,128,1,fp8,fp8,0,4.97708797454834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,24,128,1,float16,fp8,0,3.5140854517618814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,24,128,1,float16,float16,0,3.512693405151367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,24,8,128,1,float16,fp8,0,7.269936243693034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,1,128,1,float16,float16,0,3.511317253112793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,1,128,1,fp8,fp8,0,2.4569759368896484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,24,128,1,fp8,fp8,0,2.502288023630778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,2,128,1,float16,float16,0,3.6006078720092773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,1,128,1,float16,fp8,0,3.5792160034179688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,2,128,1,float16,fp8,0,3.5626614888509116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,2,128,1,fp8,fp8,0,2.4603892962137857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,4,128,1,float16,float16,0,3.5174614588419595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,4,128,1,float16,fp8,0,3.5684534708658853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,4,128,1,fp8,fp8,0,2.4669547080993652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,8,128,1,float16,float16,0,3.6024853388468423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,8,128,1,fp8,fp8,0,2.4826080004374185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,24,128,1,float16,fp8,0,1.8458879788716633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,24,128,1,float16,float16,0,1.8592480023701985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,24,8,128,1,float16,fp8,0,3.516223907470703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,1,128,1,float16,float16,0,1.8765279452006023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,1,128,1,fp8,fp8,0,1.3321653207143147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,1,128,1,float16,fp8,0,1.864672025044759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,24,128,1,fp8,fp8,0,1.353775978088379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,2,128,1,float16,float16,0,1.877562681833903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,2,128,1,float16,fp8,0,1.8576587041219075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,4,128,1,float16,float16,0,1.8814026514689128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,2,128,1,fp8,fp8,0,1.3349067370096843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,4,128,1,float16,fp8,0,1.8633012771606445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,4,128,1,fp8,fp8,0,1.3393813769022624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,8,128,1,float16,float16,0,1.918058713277181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,8,128,1,float16,fp8,0,1.8664320309956868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,24,8,128,1,fp8,fp8,0,1.3430773417154949
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,1,128,1,float16,float16,0,10.496122360229492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,1,128,1,fp8,fp8,0,6.813274383544922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,1,128,1,float16,fp8,0,10.26857058207194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,2,128,1,fp8,fp8,0,6.847967783610026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,2,128,1,float16,float16,0,10.438202540079752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,2,128,1,float16,fp8,0,10.451274871826172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,4,128,1,float16,float16,0,10.401594797770182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,4,128,1,float16,fp8,0,10.48413340250651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,4,128,1,fp8,fp8,0,6.89579709370931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,8,128,1,float16,float16,0,10.413946787516275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,24,128,1,float16,float16,0,5.24348258972168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,8,128,1,fp8,fp8,0,6.95303471883138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,1,128,1,float16,float16,0,5.100751876831055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,24,8,128,1,float16,fp8,0,10.21459706624349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,24,128,1,float16,fp8,0,5.0793759028116865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,1,128,1,float16,fp8,0,4.9239091873168945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,1,128,1,fp8,fp8,0,3.464170773824056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,2,128,1,float16,float16,0,5.151583989461263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,2,128,1,float16,fp8,0,5.060559908548991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,2,128,1,fp8,fp8,0,3.4327306747436523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,24,128,1,fp8,fp8,0,3.5173441569010415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,4,128,1,float16,float16,0,5.070746739705403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,4,128,1,fp8,fp8,0,3.4975147247314453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,4,128,1,float16,fp8,0,4.858853340148926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,8,128,1,float16,fp8,0,5.044106801350911
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,24,128,1,float16,float16,0,2.5375839869181314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,8,128,1,fp8,fp8,0,3.476938565572103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,24,8,128,1,float16,float16,0,4.940560022989909
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,24,128,1,float16,fp8,0,2.5047359466552734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,1,128,1,float16,float16,0,2.5341493288675943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,1,128,1,fp8,fp8,0,1.791279951731364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,1,128,1,float16,fp8,0,2.505882740020752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,2,128,1,float16,float16,0,2.5277652740478516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,2,128,1,float16,fp8,0,2.536250591278076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,24,128,1,fp8,fp8,0,1.8412639300028484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,2,128,1,fp8,fp8,0,1.7954026858011882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,4,128,1,float16,float16,0,2.5427145957946777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,4,128,1,float16,fp8,0,2.5137012799580893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,4,128,1,fp8,fp8,0,1.8034559885660808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,8,128,1,float16,float16,0,2.550330638885498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,8,128,1,float16,fp8,0,2.538560072580973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,24,128,1,float16,float16,0,1.3556000391642253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,24,8,128,1,fp8,fp8,0,1.8215680122375488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,1,128,1,float16,float16,0,1.365839958190918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,1,128,1,float16,fp8,0,1.3492800394694011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,1,128,1,fp8,fp8,0,0.985205332438151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,24,128,1,fp8,fp8,0,1.0039573510487874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,24,128,1,float16,fp8,0,1.3540587425231934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,2,128,1,float16,fp8,0,1.3726827303568523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,2,128,1,fp8,fp8,0,0.9882293542226156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,2,128,1,float16,float16,0,1.366869290669759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,4,128,1,float16,float16,0,1.3688799540201824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,4,128,1,fp8,fp8,0,0.9899253050486246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,4,128,1,float16,fp8,0,1.3589332898457844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,8,128,1,float16,fp8,0,1.3805813789367676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,8,128,1,fp8,fp8,0,0.9975679715474447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,24,8,128,1,float16,float16,0,1.377237319946289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,1,128,1,float16,float16,0,13.399162292480469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,1,128,1,fp8,fp8,0,9.234282811482748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,1,128,1,float16,fp8,0,13.275323232014975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,2,128,1,float16,float16,0,13.68282699584961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,2,128,1,fp8,fp8,0,9.195930480957031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,2,128,1,float16,fp8,0,13.854117075602213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,4,128,1,float16,float16,0,13.663141886393229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,4,128,1,fp8,fp8,0,9.292207717895508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,4,128,1,float16,fp8,0,13.473088582356771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,8,128,1,float16,float16,0,13.728815714518229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,8,128,1,fp8,fp8,0,9.308810551961264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,24,128,1,float16,float16,0,7.00816535949707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,24,8,128,1,float16,fp8,0,13.77561060587565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,1,128,1,float16,float16,0,6.520671844482422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,24,128,1,float16,fp8,0,6.706975936889648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,1,128,1,float16,fp8,0,6.658405303955078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,1,128,1,fp8,fp8,0,4.545616149902344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,2,128,1,float16,float16,0,6.85430908203125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,24,128,1,fp8,fp8,0,4.80514653523763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,2,128,1,float16,fp8,0,6.779114405314128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,2,128,1,fp8,fp8,0,4.570032119750977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,4,128,1,float16,float16,0,6.868560155232747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,4,128,1,float16,fp8,0,6.745354970296224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,4,128,1,fp8,fp8,0,4.622463862101237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,8,128,1,float16,float16,0,7.002021153767903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,1,128,1,float16,float16,0,3.2512534459431968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,8,128,1,fp8,fp8,0,4.6298878987630205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,24,128,1,float16,float16,0,3.3749653498331704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,24,8,128,1,float16,fp8,0,6.823429107666016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,1,128,1,float16,fp8,0,3.2847731908162436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,1,128,1,fp8,fp8,0,2.329653263092041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,24,128,1,fp8,fp8,0,2.4162880579630532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,24,128,1,float16,fp8,0,3.302330652872721
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,2,128,1,float16,float16,0,3.270719846089681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,2,128,1,fp8,fp8,0,2.3349547386169434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,2,128,1,float16,fp8,0,3.2125120162963867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,4,128,1,float16,float16,0,3.401882807413737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,4,128,1,float16,fp8,0,3.228901227315267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,4,128,1,fp8,fp8,0,2.3480052947998047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,8,128,1,float16,fp8,0,3.326197306315104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,8,128,1,fp8,fp8,0,2.3689972559611
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,24,128,1,float16,float16,0,1.709935983022054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,1,128,1,float16,float16,0,1.704319953918457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,24,8,128,1,float16,float16,0,3.2903947830200195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,24,128,1,float16,fp8,0,1.7338933944702148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,24,128,1,fp8,fp8,0,1.2741759618123372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,1,128,1,fp8,fp8,0,1.233786662419637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,1,128,1,float16,fp8,0,1.6897759437561035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,2,128,1,float16,float16,0,1.704485257466634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,2,128,1,float16,fp8,0,1.686522642771403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,2,128,1,fp8,fp8,0,1.2361866633097331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,4,128,1,float16,float16,0,1.7371519406636555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,4,128,1,float16,fp8,0,1.6911892890930176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,4,128,1,fp8,fp8,0,1.2408479849497478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,8,128,1,float16,float16,0,1.7241066296895344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,8,128,1,float16,fp8,0,1.696565310160319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,24,8,128,1,fp8,fp8,0,1.2506720225016277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,24,128,1,float16,float16,0,0.9376107056935629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,1,128,1,float16,float16,0,0.9404906431833903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,1,128,1,float16,fp8,0,0.9252266883850098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,1,128,1,fp8,fp8,0,0.6479093233744303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,24,128,1,float16,fp8,0,0.9411733150482178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,24,128,1,fp8,fp8,0,0.6701119740804037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,2,128,1,float16,float16,0,0.9371946652730306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,2,128,1,float16,fp8,0,0.9274933338165283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,2,128,1,fp8,fp8,0,0.6496426661809286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,4,128,1,float16,float16,0,0.9408000310262045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,4,128,1,float16,fp8,0,0.9281546274820963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,4,128,1,fp8,fp8,0,0.6532053152720133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,8,128,1,float16,fp8,0,0.9303306738535563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,8,128,1,float16,float16,0,0.9434879620869955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,24,8,128,1,fp8,fp8,0,0.6600319941838583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,1,128,1,float16,fp8,0,7.956293106079102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,1,128,1,float16,float16,0,7.975349426269531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,2,128,1,fp8,fp8,0,5.506469090779622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,2,128,1,float16,fp8,0,7.855530420939128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,2,128,1,float16,float16,0,8.22655995686849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,1,128,1,fp8,fp8,0,5.5122934977213545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,4,128,1,float16,float16,0,7.94920031229655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,4,128,1,fp8,fp8,0,5.530853271484375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,4,128,1,float16,fp8,0,8.078165054321289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,8,128,1,float16,float16,0,8.162480036417643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,8,128,1,float16,fp8,0,7.893807729085286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,24,128,1,float16,float16,0,3.940474510192871
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,24,8,128,1,fp8,fp8,0,5.673695882161458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,1,128,1,float16,float16,0,3.9224958419799805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,24,128,1,float16,fp8,0,3.9690987269083657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,1,128,1,fp8,fp8,0,2.7920427322387695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,1,128,1,float16,fp8,0,3.8909600575764975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,2,128,1,float16,fp8,0,3.8500534693400064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,2,128,1,float16,float16,0,3.8736801147460938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,2,128,1,fp8,fp8,0,2.7989174524943032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,24,128,1,fp8,fp8,0,2.916111946105957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,4,128,1,fp8,fp8,0,2.816469192504883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,4,128,1,float16,fp8,0,3.8526881535847983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,4,128,1,float16,float16,0,3.8501065572102866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,8,128,1,float16,float16,0,3.9353758494059243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,1,128,1,float16,float16,0,1.9623840649922688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,8,128,1,float16,fp8,0,3.9175891876220703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,24,8,128,1,fp8,fp8,0,2.846837361653646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,24,128,1,float16,float16,0,2.003114700317383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,24,128,1,float16,fp8,0,1.9802826245625813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,1,128,1,float16,fp8,0,1.9601066907246907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,24,128,1,fp8,fp8,0,1.5170399347941081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,1,128,1,fp8,fp8,0,1.4429972966512044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,2,128,1,float16,fp8,0,1.9432427088419597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,2,128,1,float16,float16,0,1.959013303120931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,2,128,1,fp8,fp8,0,1.4492746988932292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,4,128,1,fp8,fp8,0,1.4558080037434895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,4,128,1,float16,float16,0,1.9675040245056152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,4,128,1,float16,fp8,0,1.9735840161641438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,8,128,1,float16,float16,0,1.9843467076619465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,24,128,1,float16,float16,0,1.0556906859079997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,8,128,1,float16,fp8,0,1.9637173016866047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,24,8,128,1,fp8,fp8,0,1.4697866439819336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,1,128,1,float16,float16,0,1.0529066721598308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,1,128,1,float16,fp8,0,1.030234654744466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,1,128,1,fp8,fp8,0,0.7767840226491293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,2,128,1,float16,float16,0,1.0463679631551106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,24,128,1,fp8,fp8,0,0.8126986821492513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,24,128,1,float16,fp8,0,1.0409013430277507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,2,128,1,fp8,fp8,0,0.7790986696879069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,2,128,1,float16,fp8,0,1.028826634089152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,4,128,1,float16,float16,0,1.0480799674987793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,4,128,1,float16,fp8,0,1.0333973566691081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,4,128,1,fp8,fp8,0,0.7834186553955078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,8,128,1,float16,float16,0,1.053434689839681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,8,128,1,float16,fp8,0,1.0376266638437908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,24,8,128,1,fp8,fp8,0,0.7902613480885824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,1,128,1,float16,float16,0,0.589136004447937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,24,128,1,float16,fp8,0,0.5821599960327148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,1,128,1,float16,fp8,0,0.5813279946645101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,24,128,1,float16,float16,0,0.5873546600341797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,1,128,1,fp8,fp8,0,0.4151839812596639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,24,128,1,fp8,fp8,0,0.4328266779581706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,2,128,1,float16,float16,0,0.5908960103988647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,2,128,1,float16,fp8,0,0.5817546844482422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,2,128,1,fp8,fp8,0,0.41701332728068036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,4,128,1,float16,float16,0,0.591973344484965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,4,128,1,fp8,fp8,0,0.4180639982223511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,4,128,1,float16,fp8,0,0.5834826628367106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,8,128,1,float16,float16,0,0.5924799839655558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,8,128,1,float16,fp8,0,0.5873333215713501
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,24,8,128,1,fp8,fp8,0,0.42191465695699054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,1,128,1,float16,float16,0,7.87006950378418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,1,128,1,fp8,fp8,0,5.627583821614583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,1,128,1,float16,fp8,0,7.564506530761719
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,2,128,1,float16,fp8,0,7.744496027628581
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,2,128,1,float16,float16,0,7.751434961954753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,2,128,1,fp8,fp8,0,5.719178517659505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,4,128,1,float16,float16,0,7.8898131052653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,4,128,1,fp8,fp8,0,5.690442403157552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,4,128,1,float16,fp8,0,7.770554860432942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,24,128,1,float16,float16,0,4.035888036092122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,8,128,1,float16,fp8,0,7.747514724731445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,8,128,1,float16,float16,0,7.777680079142253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,24,8,128,1,fp8,fp8,0,5.76039441426595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,24,128,1,float16,fp8,0,3.8718347549438477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,1,128,1,float16,float16,0,3.8488747278849282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,1,128,1,float16,fp8,0,3.7942241032918296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,1,128,1,fp8,fp8,0,2.84281063079834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,2,128,1,fp8,fp8,0,2.8532374699910483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,2,128,1,float16,float16,0,3.748645464579264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,2,128,1,float16,fp8,0,3.7482080459594727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,24,128,1,fp8,fp8,0,3.022618611653646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,4,128,1,float16,float16,0,3.8244800567626953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,4,128,1,fp8,fp8,0,2.8747787475585938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,4,128,1,float16,fp8,0,3.7786026000976562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,8,128,1,float16,float16,0,3.8320159912109375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,8,128,1,float16,fp8,0,3.748586654663086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,1,128,1,float16,float16,0,1.9213226636250813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,24,8,128,1,fp8,fp8,0,2.9109811782836914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,24,128,1,float16,float16,0,1.9778134028116863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,24,128,1,float16,fp8,0,1.9595519701639812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,1,128,1,float16,fp8,0,1.8633653322855632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,1,128,1,fp8,fp8,0,1.4474879900614421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,2,128,1,float16,float16,0,1.9232853253682454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,24,128,1,fp8,fp8,0,1.551535924275716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,2,128,1,float16,fp8,0,1.8619573911031086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,2,128,1,fp8,fp8,0,1.452170689900716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,4,128,1,float16,float16,0,1.9099359512329102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,4,128,1,float16,fp8,0,1.8770826657613118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,4,128,1,fp8,fp8,0,1.4664053916931152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,8,128,1,float16,fp8,0,1.8913226127624512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,8,128,1,float16,float16,0,1.9518346786499023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,24,8,128,1,fp8,fp8,0,1.4895893732706706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,1,128,1,float16,float16,0,0.9920053482055664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,24,128,1,float16,float16,0,1.0227573712666829
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,1,128,1,float16,fp8,0,0.974671999613444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,24,128,1,float16,fp8,0,1.0130186875661213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,1,128,1,fp8,fp8,0,0.7647199630737305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,24,128,1,fp8,fp8,0,0.8189919789632162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,2,128,1,float16,float16,0,0.994645357131958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,2,128,1,float16,fp8,0,0.977013349533081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,2,128,1,fp8,fp8,0,0.7664480209350586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,4,128,1,float16,float16,0,0.9987839857737223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,4,128,1,float16,fp8,0,0.9818452994028727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,4,128,1,fp8,fp8,0,0.7770079771677653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,8,128,1,float16,float16,0,1.0067626635233562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,8,128,1,float16,fp8,0,0.989525318145752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,24,8,128,1,fp8,fp8,0,0.7835946877797445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,1,128,1,float16,float16,0,0.5454986492792765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,24,128,1,float16,float16,0,0.5527306795120239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,24,128,1,float16,fp8,0,0.5457813342412313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,1,128,1,float16,fp8,0,0.531829317410787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,24,128,1,fp8,fp8,0,0.4325600067774455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,1,128,1,fp8,fp8,0,0.3987040122350057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,2,128,1,float16,float16,0,0.5453653335571289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,2,128,1,float16,fp8,0,0.5380266507466634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,2,128,1,fp8,fp8,0,0.40116266409556073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,4,128,1,float16,float16,0,0.5468693176905314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,4,128,1,float16,fp8,0,0.5360693136850992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,8,128,1,float16,float16,0,0.5488479932149252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,4,128,1,fp8,fp8,0,0.4047573407491048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,8,128,1,float16,fp8,0,0.5410133202870687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,24,128,1,float16,float16,0,0.2945280075073242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,24,8,128,1,fp8,fp8,0,0.41414932409922284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,1,128,1,float16,float16,0,0.28566400210062665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,1,128,1,float16,fp8,0,0.28015466531117755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,24,128,1,fp8,fp8,0,0.24482667446136475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,24,128,1,float16,fp8,0,0.2906613349914551
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,1,128,1,fp8,fp8,0,0.23306665817896524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,2,128,1,float16,float16,0,0.2855520049730937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,2,128,1,float16,fp8,0,0.28195200363794964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,4,128,1,float16,float16,0,0.288373331228892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,4,128,1,float16,fp8,0,0.2804479996363322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,4,128,1,fp8,fp8,0,0.234224001566569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,8,128,1,float16,float16,0,0.2898719906806946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,2,128,1,fp8,fp8,0,0.231989324092865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,8,128,1,float16,fp8,0,0.2831520040829976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,24,8,128,1,fp8,fp8,0,0.23865600426991782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,1,128,1,float16,float16,0,4.705189387003581
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,1,128,1,float16,fp8,0,4.518357276916504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,1,128,1,fp8,fp8,0,3.60700257619222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,2,128,1,float16,float16,0,4.764533360799153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,2,128,1,fp8,fp8,0,3.624901453653971
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,2,128,1,float16,fp8,0,4.51800537109375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,4,128,1,float16,float16,0,4.864672025044759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,4,128,1,float16,fp8,0,4.552965482076009
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,4,128,1,fp8,fp8,0,3.6537707646687827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,8,128,1,float16,float16,0,4.716128031412761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,24,128,1,float16,float16,0,2.4524854024251304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,8,128,1,fp8,fp8,0,3.710458755493164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,24,128,1,float16,fp8,0,2.4576427141825357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,24,8,128,1,float16,fp8,0,4.720650672912598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,1,128,1,float16,float16,0,2.310922622680664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,1,128,1,float16,fp8,0,2.2589759826660156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,1,128,1,fp8,fp8,0,1.8215306599934895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,2,128,1,float16,float16,0,2.3157119750976562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,2,128,1,fp8,fp8,0,1.832106590270996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,2,128,1,float16,fp8,0,2.292901357014974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,4,128,1,float16,float16,0,2.3368639945983887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,24,128,1,fp8,fp8,0,1.9703572591145833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,4,128,1,float16,fp8,0,2.280442714691162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,4,128,1,fp8,fp8,0,1.8445706367492676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,8,128,1,float16,fp8,0,2.3084799448649087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,8,128,1,float16,float16,0,2.3565920193990073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,24,8,128,1,fp8,fp8,0,1.8794827461242676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,24,128,1,float16,float16,0,1.2597440083821614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,1,128,1,float16,float16,0,1.1813920338948567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,24,128,1,float16,fp8,0,1.23418132464091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,1,128,1,float16,fp8,0,1.1600746313730876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,24,128,1,fp8,fp8,0,1.0169973373413086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,1,128,1,fp8,fp8,0,0.9354826609293619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,2,128,1,float16,fp8,0,1.16047469774882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,2,128,1,float16,float16,0,1.1949493090311687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,2,128,1,fp8,fp8,0,0.9405279954274496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,4,128,1,float16,float16,0,1.1935199896494548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,4,128,1,float16,fp8,0,1.1684213479359944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,4,128,1,fp8,fp8,0,0.9471786816914877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,8,128,1,float16,float16,0,1.2040106455485027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,24,128,1,float16,float16,0,0.6559253136316935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,8,128,1,fp8,fp8,0,0.9615519841512045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,24,8,128,1,float16,fp8,0,1.1845760345458984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,1,128,1,float16,float16,0,0.6289439996083578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,1,128,1,float16,fp8,0,0.615392009417216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,24,128,1,fp8,fp8,0,0.5453706582387289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,24,128,1,float16,fp8,0,0.6472053527832031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,2,128,1,float16,float16,0,0.6318133274714152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,2,128,1,float16,fp8,0,0.6173173189163208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,1,128,1,fp8,fp8,0,0.5009919802347819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,2,128,1,fp8,fp8,0,0.5037866830825806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,4,128,1,float16,float16,0,0.6339199940363566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,4,128,1,float16,fp8,0,0.6208693186442057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,8,128,1,float16,fp8,0,0.6279733180999756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,8,128,1,float16,float16,0,0.6422986586888632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,24,128,1,float16,float16,0,0.362005352973938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,4,128,1,fp8,fp8,0,0.5079520146052042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,24,8,128,1,fp8,fp8,0,0.5138826767603556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,24,128,1,float16,fp8,0,0.35680532455444336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,1,128,1,float16,float16,0,0.3516106605529785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,1,128,1,float16,fp8,0,0.34466131528218585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,1,128,1,fp8,fp8,0,0.26585066318511963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,2,128,1,float16,float16,0,0.35331201553344727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,24,128,1,fp8,fp8,0,0.29155200719833374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,2,128,1,float16,fp8,0,0.3456053336461385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,2,128,1,fp8,fp8,0,0.26734399795532227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,4,128,1,float16,float16,0,0.3547733227411906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,4,128,1,fp8,fp8,0,0.2693866689999898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,4,128,1,float16,fp8,0,0.3486773173014323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,8,128,1,float16,fp8,0,0.35048532485961914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,8,128,1,fp8,fp8,0,0.2735946575800578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,24,8,128,1,float16,float16,0,0.3579839865366618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,1,128,1,float16,float16,0,0.18917334079742432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,24,128,1,float16,float16,0,0.1982240080833435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,24,128,1,float16,fp8,0,0.19487466414769491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,24,128,1,fp8,fp8,0,0.1688213348388672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,1,128,1,fp8,fp8,0,0.15754133462905884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,1,128,1,float16,fp8,0,0.18569600582122803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,2,128,1,float16,float16,0,0.19056000312169394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,2,128,1,float16,fp8,0,0.1855199933052063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,4,128,1,float16,float16,0,0.19138665994008383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,2,128,1,fp8,fp8,0,0.15853866934776306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,4,128,1,fp8,fp8,0,0.16058133045832315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,8,128,1,float16,float16,0,0.19266132513682047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,4,128,1,float16,fp8,0,0.18649067481358847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,8,128,1,float16,fp8,0,0.18791999419530234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,24,8,128,1,fp8,fp8,0,0.16294399897257486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,1,128,1,float16,float16,0,4.930810610453288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,1,128,1,float16,fp8,0,4.718223889668782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,2,128,1,float16,float16,0,4.915610631306966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,1,128,1,fp8,fp8,0,4.001274744669597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,2,128,1,float16,fp8,0,4.739114761352539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,2,128,1,fp8,fp8,0,4.013408025105794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,4,128,1,float16,float16,0,5.101728121439616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,4,128,1,float16,fp8,0,4.806714693705241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,4,128,1,fp8,fp8,0,4.072725296020508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,8,128,1,float16,float16,0,5.05018679300944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,8,128,1,float16,fp8,0,4.847584088643392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,24,8,128,1,fp8,fp8,0,4.116208076477051
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,1,128,1,float16,float16,0,2.4271039962768555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,24,128,1,float16,fp8,0,2.6032533645629883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,24,128,1,float16,float16,0,2.6594719886779785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,1,128,1,float16,fp8,0,2.345802625020345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,1,128,1,fp8,fp8,0,1.9947679837544758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,2,128,1,float16,float16,0,2.443530718485514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,2,128,1,float16,fp8,0,2.3554506301879883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,24,128,1,fp8,fp8,0,2.1967466672261557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,2,128,1,fp8,fp8,0,2.0051093101501465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,4,128,1,float16,float16,0,2.4677440325419107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,4,128,1,float16,fp8,0,2.375744024912516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,4,128,1,fp8,fp8,0,2.0269813537597656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,8,128,1,float16,float16,0,2.499311923980713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,1,128,1,float16,float16,0,1.215173323949178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,8,128,1,float16,fp8,0,2.410048007965088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,24,8,128,1,fp8,fp8,0,2.064314683278402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,24,128,1,float16,float16,0,1.346511999766032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,1,128,1,fp8,fp8,0,1.0079893271128337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,24,128,1,fp8,fp8,0,1.1221813360850017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,1,128,1,float16,fp8,0,1.1834186712900798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,24,128,1,float16,fp8,0,1.3012426694234211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,2,128,1,float16,fp8,0,1.1883893013000488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,2,128,1,float16,float16,0,1.218725363413493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,2,128,1,fp8,fp8,0,1.0122986634572346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,4,128,1,float16,float16,0,1.2278400262196858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,4,128,1,fp8,fp8,0,1.0247413317362468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,4,128,1,float16,fp8,0,1.1970453262329102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,8,128,1,float16,fp8,0,1.2144160270690918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,8,128,1,float16,float16,0,1.246122678120931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,1,128,1,float16,float16,0,0.6370933453241984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,24,128,1,float16,float16,0,0.6912319660186768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,24,8,128,1,fp8,fp8,0,1.046895980834961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,1,128,1,float16,fp8,0,0.6194239854812622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,24,128,1,fp8,fp8,0,0.5861599842707316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,24,128,1,float16,fp8,0,0.6693493525187174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,2,128,1,float16,float16,0,0.6374613444010416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,2,128,1,float16,fp8,0,0.620906670888265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,1,128,1,fp8,fp8,0,0.529695987701416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,2,128,1,fp8,fp8,0,0.5307146708170573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,4,128,1,float16,float16,0,0.6425173282623291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,4,128,1,float16,fp8,0,0.6265759865442911
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,4,128,1,fp8,fp8,0,0.5369386672973633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,8,128,1,float16,float16,0,0.6496906677881876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,8,128,1,float16,fp8,0,0.6355679829915365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,24,128,1,float16,float16,0,0.36640000343322754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,24,8,128,1,fp8,fp8,0,0.5477706591288248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,1,128,1,float16,float16,0,0.34461331367492676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,1,128,1,float16,fp8,0,0.3367626667022705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,1,128,1,fp8,fp8,0,0.27718933423360187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,24,128,1,float16,fp8,0,0.3605333169301351
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,24,128,1,fp8,fp8,0,0.3189706603686015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,2,128,1,float16,float16,0,0.3461013237635295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,2,128,1,float16,fp8,0,0.3395306666692098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,2,128,1,fp8,fp8,0,0.2789919972419739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,4,128,1,float16,float16,0,0.35026665528615314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,4,128,1,float16,fp8,0,0.34133867422739667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,4,128,1,fp8,fp8,0,0.281823992729187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,8,128,1,float16,float16,0,0.3526506821314494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,8,128,1,float16,fp8,0,0.34462400277455646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,24,128,1,float16,float16,0,0.2032053271929423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,1,128,1,float16,fp8,0,0.18036266167958578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,24,8,128,1,fp8,fp8,0,0.2897439996401469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,1,128,1,fp8,fp8,0,0.1585599978764852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,24,128,1,fp8,fp8,0,0.17484800020853677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,1,128,1,float16,float16,0,0.18396800756454468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,24,128,1,float16,fp8,0,0.1995840072631836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,2,128,1,float16,float16,0,0.1853813330332438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,2,128,1,float16,fp8,0,0.1811359922091166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,2,128,1,fp8,fp8,0,0.15982400377591452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,4,128,1,float16,fp8,0,0.18239466349283853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,4,128,1,float16,float16,0,0.1865440011024475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,4,128,1,fp8,fp8,0,0.16081066926320395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,8,128,1,float16,float16,0,0.18992000818252563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,8,128,1,float16,fp8,0,0.18492267529169717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,24,8,128,1,fp8,fp8,0,0.164192001024882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,24,128,1,float16,fp8,0,0.11475200454394023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,24,128,1,fp8,fp8,0,0.10656533638636272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,1,128,1,float16,fp8,0,0.10839999715487163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,24,128,1,float16,float16,0,0.11698666214942932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,1,128,1,fp8,fp8,0,0.09572266538937886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,1,128,1,float16,float16,0,0.11025066177050273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,2,128,1,float16,float16,0,0.11105066537857056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,2,128,1,float16,fp8,0,0.10935466488202412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,2,128,1,fp8,fp8,0,0.09636800487836202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,4,128,1,float16,float16,0,0.11181333661079407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,4,128,1,fp8,fp8,0,0.09820266564687093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,8,128,1,float16,fp8,0,0.11153599619865417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,4,128,1,float16,fp8,0,0.10912533601125081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,8,128,1,fp8,fp8,0,0.10108799735705058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,24,8,128,1,float16,float16,0,0.11264533797899882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,1,128,1,float16,float16,0,3.160773277282715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,1,128,1,float16,fp8,0,3.031599998474121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,1,128,1,fp8,fp8,0,2.660048007965088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,2,128,1,fp8,fp8,0,2.67250124613444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,2,128,1,float16,float16,0,3.1727094650268555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,2,128,1,float16,fp8,0,3.043717384338379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,4,128,1,float16,float16,0,3.2222185134887695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,4,128,1,fp8,fp8,0,2.725616137186686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,4,128,1,float16,fp8,0,3.0867093404134116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,24,128,1,float16,float16,0,1.7459306716918945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,8,128,1,float16,fp8,0,3.1410080591837564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,8,128,1,fp8,fp8,0,2.778282801310221
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,24,8,128,1,float16,float16,0,3.266688028971354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,24,128,1,float16,fp8,0,1.6942507425944011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,1,128,1,float16,float16,0,1.566746711730957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,1,128,1,float16,fp8,0,1.5124746958414714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,1,128,1,fp8,fp8,0,1.3391520182291667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,2,128,1,float16,float16,0,1.5754720369974773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,2,128,1,float16,fp8,0,1.5171786944071453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,2,128,1,fp8,fp8,0,1.3440906206766765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,24,128,1,fp8,fp8,0,1.5059626897176106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,4,128,1,float16,float16,0,1.5915625890096028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,4,128,1,float16,fp8,0,1.5343573888142903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,4,128,1,fp8,fp8,0,1.3683360417683919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,8,128,1,float16,float16,0,1.6250613530476887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,8,128,1,float16,fp8,0,1.568394660949707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,24,8,128,1,fp8,fp8,0,1.3977972666422527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,24,128,1,float16,float16,0,0.8848640124003092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,24,128,1,float16,fp8,0,0.8612799644470215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,1,128,1,float16,float16,0,0.7978026866912842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,1,128,1,float16,fp8,0,0.7727733453114828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,1,128,1,fp8,fp8,0,0.6843573252360026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,2,128,1,float16,float16,0,0.7996266682942709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,2,128,1,float16,fp8,0,0.7762293020884196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,2,128,1,fp8,fp8,0,0.6870773633321127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,4,128,1,float16,float16,0,0.8093120257059733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,24,128,1,fp8,fp8,0,0.7727306683858236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,4,128,1,float16,fp8,0,0.7839413483937582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,4,128,1,fp8,fp8,0,0.6964853604634603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,8,128,1,float16,float16,0,0.8180639743804932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,8,128,1,float16,fp8,0,0.7989066441853842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,24,128,1,float16,float16,0,0.46540268262227374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,1,128,1,float16,float16,0,0.42137598991394043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,24,8,128,1,fp8,fp8,0,0.7137173016866049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,24,128,1,float16,fp8,0,0.4518666664759318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,1,128,1,float16,fp8,0,0.4097226858139038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,24,128,1,fp8,fp8,0,0.4081333478291829
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,1,128,1,fp8,fp8,0,0.3646719853083293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,2,128,1,float16,float16,0,0.4251999855041504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,2,128,1,fp8,fp8,0,0.3662453492482503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,2,128,1,float16,fp8,0,0.4107786814371745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,4,128,1,fp8,fp8,0,0.3695360024770101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,4,128,1,float16,float16,0,0.42815999190012616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,4,128,1,float16,fp8,0,0.41570667425791424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,8,128,1,float16,float16,0,0.4330559968948364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,8,128,1,float16,fp8,0,0.424453337987264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,24,8,128,1,fp8,fp8,0,0.3779253164927165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,24,128,1,float16,float16,0,0.2506879965464274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,1,128,1,float16,fp8,0,0.22649067640304565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,24,128,1,float16,fp8,0,0.2456106742223104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,1,128,1,float16,float16,0,0.23274133602778116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,1,128,1,fp8,fp8,0,0.19501332441965738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,24,128,1,fp8,fp8,0,0.22283732891082764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,2,128,1,float16,float16,0,0.2336746652921041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,2,128,1,float16,fp8,0,0.22742400566736856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,2,128,1,fp8,fp8,0,0.19646932681401572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,4,128,1,float16,float16,0,0.23634666204452515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,4,128,1,fp8,fp8,0,0.19793067375818887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,4,128,1,float16,fp8,0,0.23000532388687134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,24,128,1,float16,float16,0,0.1418186624844869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,8,128,1,fp8,fp8,0,0.20243199666341147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,8,128,1,float16,fp8,0,0.23425066471099854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,24,8,128,1,float16,float16,0,0.23976532618204752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,24,128,1,float16,fp8,0,0.13991999626159668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,1,128,1,float16,fp8,0,0.12479999661445618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,24,128,1,fp8,fp8,0,0.12743467092514038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,1,128,1,float16,float16,0,0.12756266196568808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,1,128,1,fp8,fp8,0,0.11364266276359558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,2,128,1,float16,float16,0,0.12825066844622293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,2,128,1,fp8,fp8,0,0.11478933691978455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,2,128,1,float16,fp8,0,0.12528533736864725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,4,128,1,float16,float16,0,0.12924266854921976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,4,128,1,float16,fp8,0,0.12738133470217386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,8,128,1,float16,float16,0,0.13153599699338278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,4,128,1,fp8,fp8,0,0.11626133322715759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,8,128,1,float16,fp8,0,0.1283253331979116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,24,8,128,1,fp8,fp8,0,0.11898133158683777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,24,128,1,float16,float16,0,0.08518399794896443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,1,128,1,float16,float16,0,0.07981333136558533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,1,128,1,float16,fp8,0,0.07871466875076294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,24,128,1,float16,fp8,0,0.08433066805203755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,24,128,1,fp8,fp8,0,0.08025600016117096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,1,128,1,fp8,fp8,0,0.07092266778151195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,2,128,1,float16,float16,0,0.07994133234024048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,2,128,1,fp8,fp8,0,0.0705866664648056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,2,128,1,float16,fp8,0,0.07897600034872691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,4,128,1,float16,fp8,0,0.07879999776681264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,8,128,1,float16,float16,0,0.08125333487987518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,4,128,1,fp8,fp8,0,0.0716480016708374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,8,128,1,float16,fp8,0,0.07924800117810567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,4,128,1,float16,float16,0,0.07996266583601634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,24,8,128,1,fp8,fp8,0,0.07309333483378093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,1,128,1,float16,fp8,0,2.9194186528523765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,1,128,1,float16,float16,0,2.9251092274983725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,2,128,1,fp8,fp8,0,2.752431869506836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,1,128,1,fp8,fp8,0,2.6291786829630532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,2,128,1,float16,fp8,0,2.951706568400065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,2,128,1,float16,float16,0,3.018085479736328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,4,128,1,float16,float16,0,3.1725279490152993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,4,128,1,float16,fp8,0,3.1966241200764975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,4,128,1,fp8,fp8,0,2.971808115641276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,24,128,1,float16,float16,0,1.913658618927002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,8,128,1,float16,float16,0,3.128373463948568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,8,128,1,float16,fp8,0,3.2517760594685874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,24,8,128,1,fp8,fp8,0,2.9977814356486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,1,128,1,float16,float16,0,1.4853866895039876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,24,128,1,float16,fp8,0,1.834997336069743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,1,128,1,float16,fp8,0,1.4761226971944172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,1,128,1,fp8,fp8,0,1.3238080342610676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,2,128,1,float16,float16,0,1.499077320098877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,2,128,1,float16,fp8,0,1.4771679242451985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,2,128,1,fp8,fp8,0,1.3818772633870442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,24,128,1,fp8,fp8,0,1.617087999979655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,4,128,1,float16,float16,0,1.5852640469868977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,4,128,1,float16,fp8,0,1.5840160051981609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,4,128,1,fp8,fp8,0,1.4937814076741536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,8,128,1,float16,fp8,0,1.6183093388875325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,8,128,1,float16,float16,0,1.5623680750528972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,1,128,1,float16,float16,0,0.7558346589406332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,24,128,1,float16,float16,0,0.9625493685404459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,24,8,128,1,fp8,fp8,0,1.5120426813761394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,24,128,1,float16,fp8,0,0.924079974492391
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,1,128,1,float16,fp8,0,0.753701368967692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,1,128,1,fp8,fp8,0,0.6634560028711954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,24,128,1,fp8,fp8,0,0.8127733071645101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,2,128,1,float16,float16,0,0.7626720269521078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,2,128,1,float16,fp8,0,0.7571520010630289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,2,128,1,fp8,fp8,0,0.6983946959177653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,4,128,1,fp8,fp8,0,0.7580320040384928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,4,128,1,float16,fp8,0,0.7854560216267904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,4,128,1,float16,float16,0,0.7938346862792969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,8,128,1,float16,float16,0,0.8010026613871256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,1,128,1,float16,float16,0,0.3938666582107544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,8,128,1,float16,fp8,0,0.7967999776204427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,24,8,128,1,fp8,fp8,0,0.7649386723836263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,1,128,1,float16,fp8,0,0.3920373519261678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,24,128,1,float16,float16,0,0.4898080031077067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,24,128,1,float16,fp8,0,0.47089068094889325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,1,128,1,fp8,fp8,0,0.3457119862238566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,24,128,1,fp8,fp8,0,0.41837334632873535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,2,128,1,float16,fp8,0,0.39457066853841144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,2,128,1,float16,float16,0,0.39794667561848956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,2,128,1,fp8,fp8,0,0.3521226644515991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,4,128,1,float16,fp8,0,0.4028960069020589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,4,128,1,fp8,fp8,0,0.39109333356221515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,4,128,1,float16,float16,0,0.40989331404368085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,8,128,1,float16,float16,0,0.41473066806793213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,8,128,1,float16,fp8,0,0.4131893316904704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,24,8,128,1,fp8,fp8,0,0.3924533526102702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,1,128,1,float16,fp8,0,0.21145067612330118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,24,128,1,float16,float16,0,0.2589973409970601
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,1,128,1,float16,float16,0,0.21140267451604208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,24,128,1,float16,fp8,0,0.24994667371114096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,24,128,1,fp8,fp8,0,0.22127999862035116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,1,128,1,fp8,fp8,0,0.1775146722793579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,2,128,1,float16,float16,0,0.21192532777786255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,2,128,1,float16,fp8,0,0.21237866083780924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,2,128,1,fp8,fp8,0,0.1795519987742106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,4,128,1,float16,fp8,0,0.2164319952329
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,4,128,1,fp8,fp8,0,0.1965386668841044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,4,128,1,float16,float16,0,0.21975467602411905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,8,128,1,float16,float16,0,0.2248693307240804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,8,128,1,float16,fp8,0,0.2214133342107137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,24,8,128,1,fp8,fp8,0,0.2007146676381429
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,24,128,1,float16,float16,0,0.15339199701944986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,24,128,1,fp8,fp8,0,0.11621866623560588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,1,128,1,float16,float16,0,0.11528533697128296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,24,128,1,float16,fp8,0,0.14933866262435913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,1,128,1,float16,fp8,0,0.11469333370526631
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,1,128,1,fp8,fp8,0,0.0983893374602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,2,128,1,float16,float16,0,0.11580800016721089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,2,128,1,float16,fp8,0,0.11583466331164043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,2,128,1,fp8,fp8,0,0.09907199939092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,4,128,1,float16,fp8,0,0.11754133303960164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,4,128,1,float16,float16,0,0.11913599570592244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,4,128,1,fp8,fp8,0,0.10757866501808167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,8,128,1,float16,float16,0,0.12170666456222534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,8,128,1,float16,fp8,0,0.12091199556986491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,24,8,128,1,fp8,fp8,0,0.10821333527565002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,24,128,1,float16,float16,0,0.07582933207352956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,24,128,1,float16,fp8,0,0.07403199871381123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,1,128,1,float16,float16,0,0.06432533264160156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,1,128,1,float16,fp8,0,0.06318933268388112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,24,128,1,fp8,fp8,0,0.0703359991312027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,1,128,1,fp8,fp8,0,0.05719999969005585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,2,128,1,float16,float16,0,0.06393066545327504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,2,128,1,float16,fp8,0,0.06417599817117055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,2,128,1,fp8,fp8,0,0.05879466732343038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,4,128,1,float16,float16,0,0.06668266654014587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,4,128,1,float16,fp8,0,0.06660800178845723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,8,128,1,float16,float16,0,0.06774400174617767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,4,128,1,fp8,fp8,0,0.06300800045331319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,8,128,1,float16,fp8,0,0.06803733110427856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,24,8,128,1,fp8,fp8,0,0.06372799972693126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,24,128,1,float16,float16,0,0.043136000633239746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,24,128,1,float16,fp8,0,0.04305600126584371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,24,128,1,fp8,fp8,0,0.040896000961462654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,1,128,1,float16,float16,0,0.03863999992609024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,1,128,1,fp8,fp8,0,0.03707200040419897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,1,128,1,float16,fp8,0,0.038506666819254555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,2,128,1,float16,float16,0,0.03934400031963984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,2,128,1,float16,fp8,0,0.03905066599448522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,2,128,1,fp8,fp8,0,0.03766933331886927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,4,128,1,float16,float16,0,0.03994133323431015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,4,128,1,fp8,fp8,0,0.03915733347336451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,4,128,1,float16,fp8,0,0.039887999494870506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,8,128,1,float16,fp8,0,0.04027199993530909
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,8,128,1,fp8,fp8,0,0.039221333960692085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,24,8,128,1,float16,float16,0,0.03991466760635376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,1,128,1,float16,float16,0,2.386751969655355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,1,128,1,float16,fp8,0,2.2769546508789062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,1,128,1,fp8,fp8,0,2.2160746256510415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,2,128,1,float16,float16,0,2.42956272761027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,2,128,1,fp8,fp8,0,2.2983627319335938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,2,128,1,float16,fp8,0,2.3495465914408364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,4,128,1,float16,float16,0,2.542400042215983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,4,128,1,float16,fp8,0,2.5696694056193032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,4,128,1,fp8,fp8,0,2.514965375264486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,24,128,1,float16,float16,0,1.5799040794372559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,8,128,1,float16,float16,0,2.5063039461771646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,8,128,1,float16,fp8,0,2.627018610636393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,24,8,128,1,fp8,fp8,0,2.5417332649230957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,24,128,1,float16,fp8,0,1.5163787206013997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,1,128,1,float16,float16,0,1.1601706345876057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,1,128,1,float16,fp8,0,1.1557066440582275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,2,128,1,float16,float16,0,1.1890079975128174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,1,128,1,fp8,fp8,0,1.1053760051727295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,2,128,1,float16,fp8,0,1.1558132966359456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,2,128,1,fp8,fp8,0,1.1818079948425293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,4,128,1,float16,float16,0,1.2736000219980876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,24,128,1,fp8,fp8,0,1.3828694025675456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,4,128,1,fp8,fp8,0,1.2676106293996174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,4,128,1,float16,fp8,0,1.2545173168182373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,8,128,1,float16,float16,0,1.2393173376719158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,1,128,1,float16,float16,0,0.5942346652348837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,8,128,1,float16,fp8,0,1.2829279899597168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,24,128,1,float16,float16,0,0.8021600246429443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,24,8,128,1,fp8,fp8,0,1.282490650812785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,1,128,1,float16,fp8,0,0.5900853474934896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,24,128,1,float16,fp8,0,0.7614080111185709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,24,128,1,fp8,fp8,0,0.7005919615427653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,1,128,1,fp8,fp8,0,0.5481813351313273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,2,128,1,float16,float16,0,0.5978879928588867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,2,128,1,float16,fp8,0,0.5922133525212606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,2,128,1,fp8,fp8,0,0.5762399832407633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,4,128,1,float16,float16,0,0.6345066626866659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,4,128,1,float16,fp8,0,0.6212693452835083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,4,128,1,fp8,fp8,0,0.641045331954956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,8,128,1,float16,float16,0,0.6334240039189657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,8,128,1,float16,fp8,0,0.6323306560516357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,1,128,1,float16,float16,0,0.30905065933863324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,24,8,128,1,fp8,fp8,0,0.6482719977696737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,24,128,1,float16,fp8,0,0.3936320145924886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,24,128,1,fp8,fp8,0,0.3588533401489258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,24,128,1,float16,float16,0,0.40987201531728107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,1,128,1,float16,fp8,0,0.30827732880910236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,1,128,1,fp8,fp8,0,0.28563199440638226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,2,128,1,float16,float16,0,0.3124319911003113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,2,128,1,fp8,fp8,0,0.2914773424466451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,2,128,1,float16,fp8,0,0.308896005153656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,4,128,1,float16,float16,0,0.32477333148320514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,4,128,1,float16,fp8,0,0.3193653424580892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,4,128,1,fp8,fp8,0,0.33077865839004517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,1,128,1,float16,float16,0,0.1653600037097931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,8,128,1,float16,float16,0,0.3293333252271016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,8,128,1,float16,fp8,0,0.32595733801523846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,24,128,1,float16,float16,0,0.2169813315073649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,24,8,128,1,fp8,fp8,0,0.33367466926574707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,24,128,1,float16,fp8,0,0.2095253268877665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,24,128,1,fp8,fp8,0,0.1900426745414734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,1,128,1,float16,fp8,0,0.1643946667512258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,1,128,1,fp8,fp8,0,0.15079999963442484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,2,128,1,float16,float16,0,0.16768000523249307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,2,128,1,float16,fp8,0,0.16594133774439493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,2,128,1,fp8,fp8,0,0.15178666512171426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,4,128,1,float16,float16,0,0.17316800355911255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,4,128,1,float16,fp8,0,0.17165333032608032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,4,128,1,fp8,fp8,0,0.17096000909805298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,8,128,1,float16,float16,0,0.1787733236948649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,8,128,1,float16,fp8,0,0.17628800868988037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,24,128,1,float16,fp8,0,0.1309760014216105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,24,128,1,float16,float16,0,0.1334826648235321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,24,8,128,1,fp8,fp8,0,0.1734773317972819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,1,128,1,float16,float16,0,0.09278399745623271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,1,128,1,float16,fp8,0,0.09337066610654195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,1,128,1,fp8,fp8,0,0.08556800087292989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,2,128,1,float16,float16,0,0.09469333291053772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,2,128,1,float16,fp8,0,0.09453866879145305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,24,128,1,fp8,fp8,0,0.10355200370152791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,2,128,1,fp8,fp8,0,0.08644266923268636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,4,128,1,float16,float16,0,0.0976106623808543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,4,128,1,float16,fp8,0,0.09722666939099629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,4,128,1,fp8,fp8,0,0.09412800272305806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,8,128,1,float16,fp8,0,0.09897067149480183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,8,128,1,float16,float16,0,0.10108799735705058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,24,128,1,float16,float16,0,0.06769066552321117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,1,128,1,float16,float16,0,0.053157334526379905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,24,8,128,1,fp8,fp8,0,0.09550399581591289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,24,128,1,float16,fp8,0,0.06531199812889099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,1,128,1,float16,fp8,0,0.053173333406448364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,24,128,1,fp8,fp8,0,0.06196266909440359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,1,128,1,fp8,fp8,0,0.04854933420817057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,2,128,1,float16,float16,0,0.05410666763782501
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,2,128,1,float16,fp8,0,0.05448000133037567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,4,128,1,float16,float16,0,0.056602666775385536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,4,128,1,float16,fp8,0,0.055946667989095054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,4,128,1,fp8,fp8,0,0.05503466725349426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,8,128,1,float16,float16,0,0.058245331048965454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,2,128,1,fp8,fp8,0,0.049626668294270836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,8,128,1,float16,fp8,0,0.057274664441744484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,24,8,128,1,fp8,fp8,0,0.055349335074424744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,24,128,1,float16,float16,0,0.039434666434923805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,1,128,1,float16,float16,0,0.03489066660404205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,24,128,1,float16,fp8,0,0.03815466662247976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,24,128,1,fp8,fp8,0,0.036981334288915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,1,128,1,fp8,fp8,0,0.0335359995563825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,1,128,1,float16,fp8,0,0.03495466709136963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,2,128,1,float16,float16,0,0.03452266752719879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,2,128,1,float16,fp8,0,0.03498666733503342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,2,128,1,fp8,fp8,0,0.033930666744709015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,4,128,1,float16,float16,0,0.03570666660865148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,4,128,1,float16,fp8,0,0.035786665976047516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,4,128,1,fp8,fp8,0,0.0352960005402565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,8,128,1,float16,fp8,0,0.036330667634805046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,8,128,1,float16,float16,0,0.03668266783157984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,24,128,1,fp8,fp8,0,0.02794666588306427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,24,128,1,float16,fp8,0,0.03017599880695343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,24,128,1,float16,float16,0,0.03036266565322876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,24,8,128,1,fp8,fp8,0,0.03517866631348928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,1,128,1,float16,float16,0,0.027637332677841187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,2,128,1,float16,float16,0,0.027978666126728058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,2,128,1,float16,fp8,0,0.028437333802382152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,1,128,1,float16,fp8,0,0.028079998989899952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,2,128,1,fp8,fp8,0,0.026885333160559338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,4,128,1,float16,float16,0,0.028357334434986115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,4,128,1,float16,fp8,0,0.02874133239189784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,1,128,1,fp8,fp8,0,0.026730666557947796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,4,128,1,fp8,fp8,0,0.027503999571005504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,8,128,1,float16,float16,0,0.02899733434120814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,8,128,1,float16,fp8,0,0.029088000456492107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,24,8,128,1,fp8,fp8,0,0.02796799937884013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,1,128,1,float16,float16,0,0.9984587033589681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,1,128,1,float16,fp8,0,0.9888479709625244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,2,128,1,fp8,fp8,0,0.9905546506245931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,2,128,1,float16,fp8,0,0.9976906776428223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,1,128,1,fp8,fp8,0,0.956816037495931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,2,128,1,float16,float16,0,1.0315626462300618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,4,128,1,float16,float16,0,1.116751988728841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,4,128,1,float16,fp8,0,1.0938133398691814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,4,128,1,fp8,fp8,0,1.1204319794972737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,24,128,1,float16,fp8,0,0.7006026903788248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,8,128,1,float16,float16,0,1.1138346989949544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,24,128,1,float16,float16,0,0.7308586438496908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,8,128,1,float16,fp8,0,1.1354186534881592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,24,8,128,1,fp8,fp8,0,1.1436853408813477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,1,128,1,float16,float16,0,0.5109173456827799
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,1,128,1,float16,fp8,0,0.5035519997278849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,1,128,1,fp8,fp8,0,0.4751146634419759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,2,128,1,float16,fp8,0,0.5123146772384644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,2,128,1,float16,float16,0,0.5182826519012451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,2,128,1,fp8,fp8,0,0.49781866868336994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,4,128,1,float16,float16,0,0.5583200057347616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,24,128,1,fp8,fp8,0,0.6336426734924316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,4,128,1,float16,fp8,0,0.5337066650390625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,4,128,1,fp8,fp8,0,0.571290651957194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,8,128,1,float16,float16,0,0.5553066730499268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,8,128,1,float16,fp8,0,0.55403733253479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,24,8,128,1,fp8,fp8,0,0.5781386693318685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,24,128,1,float16,float16,0,0.3758133252461751
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,1,128,1,float16,float16,0,0.2665226658185323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,24,128,1,float16,fp8,0,0.3569120168685913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,1,128,1,float16,fp8,0,0.2648959954579671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,24,128,1,fp8,fp8,0,0.32390399773915607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,2,128,1,float16,fp8,0,0.2685439984003703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,2,128,1,fp8,fp8,0,0.2566666603088379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,1,128,1,fp8,fp8,0,0.2508479952812195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,2,128,1,float16,float16,0,0.2728053331375122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,4,128,1,float16,float16,0,0.28416534264882404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,4,128,1,float16,fp8,0,0.2787253260612488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,4,128,1,fp8,fp8,0,0.29421865940093994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,8,128,1,float16,fp8,0,0.2853813370068868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,8,128,1,float16,float16,0,0.29082133372624713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,1,128,1,float16,float16,0,0.1439253290494283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,24,8,128,1,fp8,fp8,0,0.29621867338816327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,24,128,1,float16,float16,0,0.20133866866429648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,24,128,1,float16,fp8,0,0.19184533754984537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,24,128,1,fp8,fp8,0,0.17291200160980225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,1,128,1,float16,fp8,0,0.14272532860438028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,1,128,1,fp8,fp8,0,0.13608533143997192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,2,128,1,float16,float16,0,0.1457919975121816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,4,128,1,float16,fp8,0,0.150325338045756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,2,128,1,fp8,fp8,0,0.13879467050234476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,4,128,1,float16,float16,0,0.15204800168673197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,2,128,1,float16,fp8,0,0.14426133036613464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,4,128,1,fp8,fp8,0,0.15363732973734537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,8,128,1,float16,float16,0,0.15691733360290527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,8,128,1,float16,fp8,0,0.15566399693489075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,1,128,1,float16,float16,0,0.08388266960779826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,24,8,128,1,fp8,fp8,0,0.15616533160209656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,24,128,1,float16,fp8,0,0.1179200013478597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,24,128,1,float16,float16,0,0.12172800302505493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,1,128,1,float16,fp8,0,0.08364799618721008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,24,128,1,fp8,fp8,0,0.09570667147636414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,1,128,1,fp8,fp8,0,0.07703466713428497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,2,128,1,float16,fp8,0,0.08566932876904805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,2,128,1,float16,float16,0,0.08518399794896443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,2,128,1,fp8,fp8,0,0.07825066645940144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,4,128,1,float16,fp8,0,0.08823466300964355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,4,128,1,float16,float16,0,0.08878933389981587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,8,128,1,float16,float16,0,0.0916319986184438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,4,128,1,fp8,fp8,0,0.08538666367530823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,8,128,1,float16,fp8,0,0.09016533692677815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,24,128,1,float16,float16,0,0.06374399860699971
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,24,8,128,1,fp8,fp8,0,0.08657067020734151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,24,128,1,float16,fp8,0,0.06152533491452535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,1,128,1,float16,float16,0,0.04872000217437744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,24,128,1,fp8,fp8,0,0.05738666653633118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,1,128,1,fp8,fp8,0,0.04287466903527578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,2,128,1,float16,float16,0,0.04946133494377136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,1,128,1,float16,fp8,0,0.04887466629346212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,2,128,1,float16,fp8,0,0.05009600023428599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,2,128,1,fp8,fp8,0,0.044821331898371376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,4,128,1,float16,float16,0,0.05154666801293691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,4,128,1,fp8,fp8,0,0.04817600051561991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,4,128,1,float16,fp8,0,0.050800000627835594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,8,128,1,float16,float16,0,0.05293866495291392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,8,128,1,fp8,fp8,0,0.050442665815353394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,24,128,1,float16,float16,0,0.037050666908423104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,24,8,128,1,float16,fp8,0,0.05343466500441233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,24,128,1,float16,fp8,0,0.037045332292715706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,1,128,1,float16,fp8,0,0.03373866776625315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,24,128,1,fp8,fp8,0,0.0351946676770846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,1,128,1,fp8,fp8,0,0.030271999537944794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,2,128,1,float16,float16,0,0.033957332372665405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,1,128,1,float16,float16,0,0.033759998778502144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,2,128,1,float16,fp8,0,0.034346667428811394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,2,128,1,fp8,fp8,0,0.03109866629044215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,4,128,1,float16,float16,0,0.03449599941571554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,4,128,1,float16,fp8,0,0.03457066665093104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,4,128,1,fp8,fp8,0,0.03216533362865448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,24,128,1,float16,float16,0,0.026954665780067444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,8,128,1,fp8,fp8,0,0.03305066625277201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,8,128,1,float16,fp8,0,0.03597866743803024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,24,8,128,1,float16,float16,0,0.03605866680542628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,1,128,1,float16,float16,0,0.024869332710901897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,24,128,1,fp8,fp8,0,0.02508266766866048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,1,128,1,float16,fp8,0,0.025216000775496166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,24,128,1,float16,fp8,0,0.027024000883102417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,1,128,1,fp8,fp8,0,0.02330133318901062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,2,128,1,fp8,fp8,0,0.02327999969323476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,4,128,1,float16,fp8,0,0.026165333886941273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,4,128,1,float16,float16,0,0.02587733417749405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,2,128,1,float16,float16,0,0.025578667720158894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,4,128,1,fp8,fp8,0,0.02436800052722295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,8,128,1,float16,float16,0,0.025701334079106648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,2,128,1,float16,fp8,0,0.02532800038655599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,8,128,1,fp8,fp8,0,0.02443733314673106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,24,8,128,1,float16,fp8,0,0.02603733291228612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,24,128,1,float16,float16,0,0.021770666042963665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,1,128,1,float16,float16,0,0.021370666722456615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,24,128,1,fp8,fp8,0,0.020234666764736176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,24,128,1,float16,fp8,0,0.02271999915440877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,1,128,1,float16,fp8,0,0.0215786670645078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,1,128,1,fp8,fp8,0,0.020106667031844456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,2,128,1,float16,float16,0,0.02160000056028366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,2,128,1,float16,fp8,0,0.02199466774861018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,2,128,1,fp8,fp8,0,0.020143999407688778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,4,128,1,float16,float16,0,0.021141332884629566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,8,128,1,float16,float16,0,0.02143466720978419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,8,128,1,float16,fp8,0,0.02186666677395503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,4,128,1,fp8,fp8,0,0.020106667031844456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,4,128,1,float16,fp8,0,0.022272000710169475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,24,8,128,1,fp8,fp8,0,0.020501332978407543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,1,128,1,float16,float16,0,0.4649706681569417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,1,128,1,float16,fp8,0,0.46555201212565106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,1,128,1,fp8,fp8,0,0.47364266713460285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,2,128,1,float16,float16,0,0.4773706595102946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,2,128,1,float16,fp8,0,0.47036266326904297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,2,128,1,fp8,fp8,0,0.5021706819534302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,4,128,1,float16,float16,0,0.5131786664326986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,4,128,1,fp8,fp8,0,0.5676853259404501
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,4,128,1,float16,fp8,0,0.5119786659876505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,8,128,1,float16,float16,0,0.5156160195668539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,8,128,1,float16,fp8,0,0.5148640076319376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,24,128,1,float16,float16,0,0.38433067003885907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,24,128,1,float16,fp8,0,0.3698666493097941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,24,8,128,1,fp8,fp8,0,0.5791840155919393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,1,128,1,float16,fp8,0,0.24701867500940958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,1,128,1,fp8,fp8,0,0.24736533562342325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,1,128,1,float16,float16,0,0.24632000923156738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,2,128,1,float16,fp8,0,0.2491040031115214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,2,128,1,fp8,fp8,0,0.2581653396288554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,2,128,1,float16,float16,0,0.2532426714897156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,24,128,1,fp8,fp8,0,0.3246399958928426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,4,128,1,float16,float16,0,0.26894932985305786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,4,128,1,float16,fp8,0,0.26126933097839355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,4,128,1,fp8,fp8,0,0.29412267605463666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,8,128,1,float16,float16,0,0.2752426664034526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,24,128,1,float16,float16,0,0.20675732692082724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,8,128,1,fp8,fp8,0,0.2975733280181885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,24,8,128,1,float16,fp8,0,0.27058666944503784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,1,128,1,float16,fp8,0,0.1350986659526825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,1,128,1,float16,float16,0,0.1361066699028015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,1,128,1,fp8,fp8,0,0.13619200388590494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,24,128,1,fp8,fp8,0,0.16991466283798218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,24,128,1,float16,fp8,0,0.19892799854278564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,2,128,1,float16,float16,0,0.13886933525403342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,2,128,1,fp8,fp8,0,0.13703466455141702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,4,128,1,float16,float16,0,0.14589866995811462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,2,128,1,float16,fp8,0,0.13760000467300415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,4,128,1,float16,fp8,0,0.1436746617158254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,4,128,1,fp8,fp8,0,0.1520799994468689
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,8,128,1,float16,float16,0,0.1530026694138845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,1,128,1,float16,float16,0,0.07861333092053731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,8,128,1,float16,fp8,0,0.14943466583887735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,24,8,128,1,fp8,fp8,0,0.1565013329188029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,24,128,1,float16,float16,0,0.11246933539708455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,24,128,1,fp8,fp8,0,0.09546132882436116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,1,128,1,float16,fp8,0,0.07877866427103679
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,24,128,1,float16,fp8,0,0.10975466171900432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,1,128,1,fp8,fp8,0,0.07558399935563405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,2,128,1,float16,fp8,0,0.07972800234953563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,2,128,1,fp8,fp8,0,0.0774293343226115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,4,128,1,float16,float16,0,0.0853653351465861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,2,128,1,float16,float16,0,0.08065066734949748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,4,128,1,float16,fp8,0,0.084197332461675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,4,128,1,fp8,fp8,0,0.08638933300971985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,24,128,1,float16,float16,0,0.06178666651248932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,8,128,1,float16,float16,0,0.08649067083994548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,8,128,1,float16,fp8,0,0.08503466844558716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,24,128,1,float16,fp8,0,0.05946133534113566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,1,128,1,float16,float16,0,0.04619733492533366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,24,128,1,fp8,fp8,0,0.05645333230495453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,24,8,128,1,fp8,fp8,0,0.08589866757392883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,1,128,1,float16,fp8,0,0.045941332976023354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,2,128,1,float16,float16,0,0.046666666865348816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,1,128,1,fp8,fp8,0,0.04327466587225596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,4,128,1,float16,float16,0,0.04923200110594431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,2,128,1,float16,fp8,0,0.048021331429481506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,2,128,1,fp8,fp8,0,0.04403733213742574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,4,128,1,float16,fp8,0,0.04892266790072123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,4,128,1,fp8,fp8,0,0.04875733455022176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,8,128,1,float16,float16,0,0.050000001986821495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,8,128,1,fp8,fp8,0,0.050160000721613564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,24,8,128,1,float16,fp8,0,0.05026666820049286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,1,128,1,float16,float16,0,0.03236266722281774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,24,128,1,fp8,fp8,0,0.03402133285999298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,1,128,1,float16,fp8,0,0.03328000009059906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,24,128,1,float16,float16,0,0.03619733452796936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,24,128,1,float16,fp8,0,0.03527999917666117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,1,128,1,fp8,fp8,0,0.03084266682465871
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,2,128,1,float16,float16,0,0.03298133363326391
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,2,128,1,float16,fp8,0,0.03297599901755651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,2,128,1,fp8,fp8,0,0.0308746670683225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,4,128,1,float16,float16,0,0.034517332911491394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,8,128,1,float16,float16,0,0.034634667138258614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,4,128,1,fp8,fp8,0,0.03245333333810171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,8,128,1,float16,fp8,0,0.03446933378775915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,4,128,1,float16,fp8,0,0.03443733354409536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,24,8,128,1,fp8,fp8,0,0.032485333581765495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,24,128,1,fp8,fp8,0,0.024735999604066212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,24,128,1,float16,fp8,0,0.025749333202838898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,24,128,1,float16,float16,0,0.026202666262785595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,1,128,1,fp8,fp8,0,0.02313599983851115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,2,128,1,float16,fp8,0,0.024383999407291412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,2,128,1,float16,float16,0,0.024490666886170704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,1,128,1,float16,float16,0,0.026911998788515728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,1,128,1,float16,fp8,0,0.024341332415739696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,2,128,1,fp8,fp8,0,0.023205332458019257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,4,128,1,float16,float16,0,0.025237334271272022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,4,128,1,fp8,fp8,0,0.024666666984558105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,8,128,1,float16,float16,0,0.02499733368555705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,8,128,1,float16,fp8,0,0.02518933266401291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,4,128,1,float16,fp8,0,0.024725332856178284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,24,128,1,float16,float16,0,0.020138667275508244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,24,128,1,float16,fp8,0,0.020986666282018025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,24,128,1,fp8,fp8,0,0.02045866722861926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,24,8,128,1,fp8,fp8,0,0.024277334411938984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,1,128,1,float16,float16,0,0.019765333582957584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,2,128,1,fp8,fp8,0,0.020069333414236706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,1,128,1,float16,fp8,0,0.020618667205174763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,2,128,1,float16,fp8,0,0.020314666132132213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,1,128,1,fp8,fp8,0,0.020090666910012562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,2,128,1,float16,float16,0,0.020293333878119785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,4,128,1,float16,fp8,0,0.020410666863123577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,4,128,1,fp8,fp8,0,0.020474666108687718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,8,128,1,float16,float16,0,0.020986666282018025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,8,128,1,float16,fp8,0,0.02054399996995926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,4,128,1,float16,float16,0,0.02032533288002014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,24,128,1,float16,float16,0,0.01905599981546402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,24,8,128,1,fp8,fp8,0,0.02053333322207133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,24,128,1,float16,fp8,0,0.01982933282852173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,1,128,1,float16,float16,0,0.019359999646743137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,24,128,1,fp8,fp8,0,0.019258666783571243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,1,128,1,float16,fp8,0,0.019493332753578823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,2,128,1,float16,float16,0,0.019093333433071773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,2,128,1,float16,fp8,0,0.02012266715367635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,2,128,1,fp8,fp8,0,0.01933866615096728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,4,128,1,float16,fp8,0,0.019744000087181728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,4,128,1,float16,float16,0,0.01932799940307935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,1,128,1,fp8,fp8,0,0.019530666371186573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,4,128,1,fp8,fp8,0,0.01945066700379054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,8,128,1,float16,float16,0,0.018992000569899876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,8,128,1,fp8,fp8,0,0.019727999965349834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,24,8,128,1,float16,fp8,0,0.020266667008399963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,1,128,1,float16,float16,0,0.3012746572494507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,1,128,1,float16,fp8,0,0.30221333106358844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,1,128,1,fp8,fp8,0,0.3272639910380046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,2,128,1,float16,fp8,0,0.30435200532277423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,2,128,1,float16,float16,0,0.30667734146118164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,2,128,1,fp8,fp8,0,0.33589335282643634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,4,128,1,float16,float16,0,0.3222080071767171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,4,128,1,float16,fp8,0,0.31492799520492554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,4,128,1,fp8,fp8,0,0.3767626682917277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,8,128,1,float16,float16,0,0.328000009059906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,8,128,1,float16,fp8,0,0.3226720094680786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,24,128,1,float16,float16,0,0.20480000972747803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,24,128,1,float16,fp8,0,0.19735999902089438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,24,8,128,1,fp8,fp8,0,0.3801493247350057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,1,128,1,float16,float16,0,0.1601706643899282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,24,128,1,fp8,fp8,0,0.20774400234222412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,1,128,1,fp8,fp8,0,0.1735573410987854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,1,128,1,float16,fp8,0,0.1606773336728414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,2,128,1,float16,float16,0,0.16301866372426352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,2,128,1,float16,fp8,0,0.16140266259511313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,4,128,1,float16,float16,0,0.16907199223836264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,4,128,1,float16,fp8,0,0.16713066895802817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,2,128,1,fp8,fp8,0,0.17642132441202799
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,4,128,1,fp8,fp8,0,0.19259732961654663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,8,128,1,float16,fp8,0,0.1725920041402181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,8,128,1,float16,float16,0,0.17388266324996948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,24,128,1,float16,float16,0,0.11101333300272624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,1,128,1,float16,float16,0,0.08866666754086812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,24,8,128,1,fp8,fp8,0,0.19738133748372397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,24,128,1,float16,fp8,0,0.10663466652234395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,1,128,1,float16,fp8,0,0.08913066983222961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,24,128,1,fp8,fp8,0,0.11517332990964253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,1,128,1,fp8,fp8,0,0.09593066573143005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,2,128,1,float16,fp8,0,0.09057600299517314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,4,128,1,float16,float16,0,0.09455999732017517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,2,128,1,fp8,fp8,0,0.09883733590443929
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,2,128,1,float16,float16,0,0.09033067027727763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,4,128,1,float16,fp8,0,0.09264533718427022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,8,128,1,float16,float16,0,0.09607999523480733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,4,128,1,fp8,fp8,0,0.10594666997591655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,8,128,1,float16,fp8,0,0.09540266791979472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,24,8,128,1,fp8,fp8,0,0.10665599505106609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,24,128,1,float16,float16,0,0.06275199850400288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,24,128,1,fp8,fp8,0,0.06628799935181935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,24,128,1,float16,fp8,0,0.06015466650327047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,1,128,1,float16,float16,0,0.05087999999523163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,2,128,1,float16,float16,0,0.052069331208864846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,1,128,1,fp8,fp8,0,0.054058666030565895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,1,128,1,float16,fp8,0,0.05133333305517832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,2,128,1,float16,fp8,0,0.052255998055140175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,2,128,1,fp8,fp8,0,0.05641599992911021
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,4,128,1,float16,fp8,0,0.054378668467203774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,4,128,1,float16,float16,0,0.05429866909980774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,4,128,1,fp8,fp8,0,0.06010133524735769
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,8,128,1,float16,float16,0,0.05462933580080668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,24,128,1,float16,float16,0,0.03516799956560135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,8,128,1,fp8,fp8,0,0.06201066573460897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,24,8,128,1,float16,fp8,0,0.05498133103052775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,24,128,1,float16,fp8,0,0.03436800092458725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,24,128,1,fp8,fp8,0,0.03792533278465271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,1,128,1,float16,float16,0,0.03323733309904734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,1,128,1,float16,fp8,0,0.03340800106525421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,1,128,1,fp8,fp8,0,0.03436800092458725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,2,128,1,fp8,fp8,0,0.03440533330043157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,2,128,1,float16,fp8,0,0.03323733309904734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,2,128,1,float16,float16,0,0.03364799916744232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,4,128,1,float16,float16,0,0.03375466664632162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,4,128,1,float16,fp8,0,0.03405333310365677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,4,128,1,fp8,fp8,0,0.03603733330965042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,8,128,1,float16,float16,0,0.03436266630887985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,8,128,1,float16,fp8,0,0.03489600121974945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,1,128,1,float16,float16,0,0.026133333643277485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,24,128,1,float16,fp8,0,0.02754133443037669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,24,128,1,fp8,fp8,0,0.028405333558718365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,1,128,1,float16,fp8,0,0.02622933437426885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,1,128,1,fp8,fp8,0,0.027082666754722595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,24,128,1,float16,float16,0,0.027679999669392902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,24,8,128,1,fp8,fp8,0,0.03625600039958954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,2,128,1,float16,float16,0,0.026506667335828144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,4,128,1,float16,float16,0,0.02647999922434489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,2,128,1,float16,fp8,0,0.02659733345111211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,2,128,1,fp8,fp8,0,0.027232001225153606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,4,128,1,float16,fp8,0,0.027424000203609467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,8,128,1,float16,fp8,0,0.027658666173617046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,4,128,1,fp8,fp8,0,0.028880000114440918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,8,128,1,fp8,fp8,0,0.028192001084486645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,24,8,128,1,float16,float16,0,0.027813332776228588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,24,128,1,float16,fp8,0,0.020080000162124634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,24,128,1,fp8,fp8,0,0.020400000115235645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,24,128,1,float16,float16,0,0.0201706662774086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,1,128,1,float16,float16,0,0.01952533299724261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,1,128,1,fp8,fp8,0,0.019519999623298645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,2,128,1,fp8,fp8,0,0.01979200045267741
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,1,128,1,float16,fp8,0,0.019882666567961376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,2,128,1,float16,fp8,0,0.019258666783571243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,4,128,1,float16,float16,0,0.01929066702723503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,2,128,1,float16,float16,0,0.01956266661485036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,4,128,1,float16,fp8,0,0.019866666446129482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,8,128,1,float16,float16,0,0.019258666783571243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,4,128,1,fp8,fp8,0,0.019882666567961376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,8,128,1,fp8,fp8,0,0.02041600023706754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,24,8,128,1,float16,fp8,0,0.01987733319401741
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,24,128,1,fp8,fp8,0,0.019146667172511418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,1,128,1,float16,float16,0,0.018245333184798557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,24,128,1,float16,fp8,0,0.019029332945744198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,24,128,1,float16,float16,0,0.018437333405017853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,1,128,1,float16,fp8,0,0.01836799954374631
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,1,128,1,fp8,fp8,0,0.01915733392039935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,2,128,1,float16,float16,0,0.018053332964579265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,2,128,1,float16,fp8,0,0.018250666558742523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,2,128,1,fp8,fp8,0,0.019189332922299702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,8,128,1,float16,float16,0,0.01858666663368543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,4,128,1,fp8,fp8,0,0.019226666539907455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,4,128,1,float16,float16,0,0.018522666146357853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,24,128,1,float16,float16,0,0.017674667139848072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,8,128,1,fp8,fp8,0,0.019381333142518997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,4,128,1,float16,fp8,0,0.01897066707412402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,24,128,1,float16,fp8,0,0.01798933371901512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,24,8,128,1,float16,fp8,0,0.018816000471512478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,24,128,1,fp8,fp8,0,0.018709332992633183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,1,128,1,float16,fp8,0,0.018426666657129925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,2,128,1,float16,fp8,0,0.01810666670401891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,1,128,1,fp8,fp8,0,0.01865600049495697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,1,128,1,float16,float16,0,0.01811733345190684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,4,128,1,float16,float16,0,0.017664000391960144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,2,128,1,fp8,fp8,0,0.019050666441520054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,2,128,1,float16,float16,0,0.017914666483799618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,4,128,1,fp8,fp8,0,0.01868266612291336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,4,128,1,float16,fp8,0,0.018250666558742523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,8,128,1,float16,float16,0,0.01758933315674464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,8,128,1,fp8,fp8,0,0.019120000302791595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,24,8,128,1,float16,fp8,0,0.01815466706951459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,1,128,1,fp8,fp8,0,0.2632533311843872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,1,128,1,float16,float16,0,0.20971200863520303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,2,128,1,float16,float16,0,0.21263466278711954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,2,128,1,float16,fp8,0,0.21179733673731485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,1,128,1,float16,fp8,0,0.20930665731430054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,2,128,1,fp8,fp8,0,0.26578666766484577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,4,128,1,float16,float16,0,0.22193600734074911
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,4,128,1,float16,fp8,0,0.2181439995765686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,4,128,1,fp8,fp8,0,0.2820319930712382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,24,128,1,float16,float16,0,0.13277332981427512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,8,128,1,float16,fp8,0,0.22613332668940225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,8,128,1,float16,float16,0,0.22780267397562662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,24,8,128,1,fp8,fp8,0,0.2850933273633321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,24,128,1,float16,fp8,0,0.13054399689038595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,1,128,1,float16,float16,0,0.11076266566912334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,1,128,1,float16,fp8,0,0.11110933621724446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,24,128,1,fp8,fp8,0,0.15944000085194907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,1,128,1,fp8,fp8,0,0.14410666624704996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,2,128,1,float16,float16,0,0.1120746632417043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,2,128,1,float16,fp8,0,0.11229866743087769
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,2,128,1,fp8,fp8,0,0.14482133587201437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,4,128,1,float16,fp8,0,0.11647466818491618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,4,128,1,float16,float16,0,0.11806399623552959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,4,128,1,fp8,fp8,0,0.15152532855669656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,8,128,1,float16,float16,0,0.12154133121172588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,8,128,1,float16,fp8,0,0.1204800009727478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,24,8,128,1,fp8,fp8,0,0.15366400281588236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,1,128,1,float16,float16,0,0.06122133135795593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,1,128,1,float16,fp8,0,0.06084799766540527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,24,128,1,float16,fp8,0,0.07100266714890797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,1,128,1,fp8,fp8,0,0.07796266674995422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,2,128,1,float16,float16,0,0.06214933097362518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,24,128,1,float16,float16,0,0.07451733450094859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,24,128,1,fp8,fp8,0,0.09018666545550029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,2,128,1,float16,fp8,0,0.06273599962393443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,2,128,1,fp8,fp8,0,0.07967466612656911
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,4,128,1,float16,float16,0,0.06465066472689311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,4,128,1,float16,fp8,0,0.06419200201829274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,4,128,1,fp8,fp8,0,0.08381866415341695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,8,128,1,float16,fp8,0,0.06585066517194112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,8,128,1,float16,float16,0,0.06633600095907848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,24,8,128,1,fp8,fp8,0,0.08409600456555684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,24,128,1,float16,float16,0,0.03980266551176707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,24,128,1,float16,fp8,0,0.03886933376391729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,24,128,1,fp8,fp8,0,0.05050666630268097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,1,128,1,float16,float16,0,0.03766400118668874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,1,128,1,float16,fp8,0,0.037130666275819145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,1,128,1,fp8,fp8,0,0.046181331078211464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,2,128,1,float16,float16,0,0.037871999045213066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,2,128,1,float16,fp8,0,0.03793066740036011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,2,128,1,fp8,fp8,0,0.04710933566093445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,4,128,1,float16,float16,0,0.03882133215665817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,4,128,1,float16,fp8,0,0.039162665605545044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,4,128,1,fp8,fp8,0,0.04894933104515076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,8,128,1,float16,fp8,0,0.039135999977588654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,24,128,1,float16,float16,0,0.02826133370399475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,24,128,1,float16,fp8,0,0.028304000695546467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,8,128,1,float16,float16,0,0.039066667358080544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,24,8,128,1,fp8,fp8,0,0.04861866434415182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,1,128,1,float16,float16,0,0.02681066592534383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,24,128,1,fp8,fp8,0,0.03281066566705704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,1,128,1,fp8,fp8,0,0.031370667119820915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,1,128,1,float16,fp8,0,0.026778665681680042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,2,128,1,float16,float16,0,0.026837334036827087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,2,128,1,float16,fp8,0,0.02737066646416982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,2,128,1,fp8,fp8,0,0.03176533430814743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,4,128,1,fp8,fp8,0,0.03253333270549774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,8,128,1,float16,float16,0,0.027679999669392902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,8,128,1,float16,fp8,0,0.027615999182065327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,4,128,1,float16,float16,0,0.02758399893840154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,4,128,1,float16,fp8,0,0.027786667148272198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,24,128,1,float16,float16,0,0.021359999974568684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,24,128,1,fp8,fp8,0,0.024821333587169647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,1,128,1,float16,float16,0,0.020842666427294414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,24,128,1,float16,fp8,0,0.02160000056028366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,1,128,1,fp8,fp8,0,0.023706667125225067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,24,8,128,1,fp8,fp8,0,0.03287466615438461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,1,128,1,float16,fp8,0,0.021898667017618816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,2,128,1,float16,float16,0,0.021690666675567627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,2,128,1,float16,fp8,0,0.02160533269246419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,2,128,1,fp8,fp8,0,0.02367466688156128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,4,128,1,float16,float16,0,0.020992000897725422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,4,128,1,float16,fp8,0,0.021509334444999695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,4,128,1,fp8,fp8,0,0.023936000963052113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,8,128,1,float16,float16,0,0.021776000658671062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,8,128,1,float16,fp8,0,0.02160000056028366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,24,8,128,1,fp8,fp8,0,0.024458666642506916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,24,128,1,float16,float16,0,0.018288000176350277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,24,128,1,float16,fp8,0,0.01806933308641116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,24,128,1,fp8,fp8,0,0.01995733380317688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,1,128,1,float16,fp8,0,0.01826133330663045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,2,128,1,float16,float16,0,0.0180479995906353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,2,128,1,float16,fp8,0,0.01791999985774358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,1,128,1,float16,float16,0,0.017637333522240322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,1,128,1,fp8,fp8,0,0.019167999426523846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,2,128,1,fp8,fp8,0,0.01926400015751521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,4,128,1,float16,float16,0,0.017765333255132038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,8,128,1,float16,float16,0,0.017632000148296356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,4,128,1,fp8,fp8,0,0.01958400011062622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,24,128,1,float16,float16,0,0.01699200024207433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,4,128,1,float16,fp8,0,0.01820266619324684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,24,128,1,float16,fp8,0,0.017525333911180496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,8,128,1,fp8,fp8,0,0.019280000279347103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,24,8,128,1,float16,fp8,0,0.018186666071414948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,24,128,1,fp8,fp8,0,0.019098666807015736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,1,128,1,float16,fp8,0,0.017632000148296356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,2,128,1,float16,float16,0,0.017509333789348602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,1,128,1,float16,float16,0,0.017173333714405697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,1,128,1,fp8,fp8,0,0.01889066646496455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,2,128,1,float16,fp8,0,0.017530667285124462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,2,128,1,fp8,fp8,0,0.01911466692884763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,4,128,1,float16,float16,0,0.01714133347074191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,4,128,1,float16,fp8,0,0.017722666263580322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,4,128,1,fp8,fp8,0,0.019205333044131596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,8,128,1,fp8,fp8,0,0.01871466636657715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,8,128,1,float16,float16,0,0.017370666066805523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,24,8,128,1,float16,fp8,0,0.017445333302021027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,24,128,1,float16,fp8,0,0.016794666647911072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,24,128,1,fp8,fp8,0,0.018437333405017853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,1,128,1,float16,float16,0,0.017055999487638474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,24,128,1,float16,float16,0,0.016879999389251072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,1,128,1,float16,fp8,0,0.017221332838137943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,1,128,1,fp8,fp8,0,0.01836799954374631
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,2,128,1,float16,float16,0,0.016901332885026932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,2,128,1,float16,fp8,0,0.017029333859682083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,2,128,1,fp8,fp8,0,0.01836799954374631
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,4,128,1,float16,float16,0,0.01703466723362605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,4,128,1,fp8,fp8,0,0.018746666610240936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,8,128,1,float16,float16,0,0.016719999412695568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,4,128,1,float16,fp8,0,0.01757866640885671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,8,128,1,fp8,fp8,0,0.01858666663368543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,24,8,128,1,float16,fp8,0,0.017386666188637417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,1,128,1,float16,float16,0,0.17359467347462973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,1,128,1,float16,fp8,0,0.17426133155822754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,1,128,1,fp8,fp8,0,0.23410133520762125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,2,128,1,float16,float16,0,0.1757919987042745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,2,128,1,fp8,fp8,0,0.23599465688069662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,2,128,1,float16,fp8,0,0.17628800868988037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,4,128,1,float16,float16,0,0.18108266592025757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,4,128,1,float16,fp8,0,0.18016533056894937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,4,128,1,fp8,fp8,0,0.24246400594711304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,8,128,1,float16,float16,0,0.18438400824864706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,8,128,1,float16,fp8,0,0.18382400274276733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,24,8,128,1,fp8,fp8,0,0.2437493403752645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,1,128,1,float16,float16,0,0.0939359962940216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,24,128,1,float16,fp8,0,0.0983840028444926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,24,128,1,float16,float16,0,0.10096533099810283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,24,128,1,fp8,fp8,0,0.137061337629954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,1,128,1,float16,fp8,0,0.09436266620953877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,1,128,1,fp8,fp8,0,0.12478933731714885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,2,128,1,float16,float16,0,0.09437333544095357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,2,128,1,float16,fp8,0,0.09486933549245198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,2,128,1,fp8,fp8,0,0.1267146666844686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,4,128,1,fp8,fp8,0,0.13010133306185404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,4,128,1,float16,fp8,0,0.09644266963005066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,8,128,1,float16,float16,0,0.0988213320573171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,4,128,1,float16,float16,0,0.09687999884287517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,8,128,1,float16,fp8,0,0.09817066788673401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,24,8,128,1,fp8,fp8,0,0.132533331712087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,1,128,1,float16,float16,0,0.054048001766204834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,24,128,1,float16,float16,0,0.05384000142415365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,24,128,1,float16,fp8,0,0.05333333214124044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,24,128,1,fp8,fp8,0,0.07306666672229767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,1,128,1,fp8,fp8,0,0.07042133311430614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,1,128,1,float16,fp8,0,0.05401599903901418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,2,128,1,float16,float16,0,0.05420800050099691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,2,128,1,float16,fp8,0,0.05433600147565206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,2,128,1,fp8,fp8,0,0.07102400064468384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,4,128,1,float16,float16,0,0.054272000988324486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,4,128,1,float16,fp8,0,0.05516799787680308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,4,128,1,fp8,fp8,0,0.07242666681607564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,8,128,1,float16,fp8,0,0.05579733351866404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,8,128,1,float16,float16,0,0.05499733487764994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,24,8,128,1,fp8,fp8,0,0.0727946658929189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,24,128,1,float16,float16,0,0.03508266558249792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,24,128,1,float16,fp8,0,0.03513599932193756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,1,128,1,float16,float16,0,0.03490666548411051
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,24,128,1,fp8,fp8,0,0.04454400142033895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,1,128,1,float16,fp8,0,0.03605866680542628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,2,128,1,float16,float16,0,0.03573333223660787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,2,128,1,float16,fp8,0,0.03490666548411051
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,4,128,1,float16,float16,0,0.036090667049090065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,1,128,1,fp8,fp8,0,0.04330666859944662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,2,128,1,fp8,fp8,0,0.043840001026789345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,4,128,1,float16,fp8,0,0.03629866739114126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,4,128,1,fp8,fp8,0,0.045093332727750145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,8,128,1,float16,fp8,0,0.03637866675853729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,8,128,1,float16,float16,0,0.03614933292071024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,24,8,128,1,fp8,fp8,0,0.0447626660267512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,24,128,1,float16,float16,0,0.02458133300145467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,24,128,1,float16,fp8,0,0.024890666206677754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,1,128,1,float16,float16,0,0.023599999646345775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,1,128,1,float16,fp8,0,0.024373332659403484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,24,128,1,fp8,fp8,0,0.028543998797734577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,2,128,1,float16,float16,0,0.024160000185171764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,1,128,1,fp8,fp8,0,0.028336000939210255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,2,128,1,fp8,fp8,0,0.028384000062942505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,4,128,1,float16,float16,0,0.024080000817775726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,4,128,1,float16,fp8,0,0.0249439999461174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,8,128,1,float16,float16,0,0.024400000770886738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,4,128,1,fp8,fp8,0,0.028149334092934925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,8,128,1,float16,fp8,0,0.024869332710901897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,2,128,1,float16,fp8,0,0.02455466737349828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,24,8,128,1,fp8,fp8,0,0.028949332733949024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,24,128,1,float16,fp8,0,0.02075733368595441
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,24,128,1,float16,float16,0,0.02011200040578842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,1,128,1,float16,fp8,0,0.02090666691462199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,24,128,1,fp8,fp8,0,0.0239680012067159
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,1,128,1,float16,float16,0,0.020389333367347717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,1,128,1,fp8,fp8,0,0.023823998868465424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,2,128,1,float16,float16,0,0.020687999824682873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,2,128,1,float16,fp8,0,0.021274665991465252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,2,128,1,fp8,fp8,0,0.02362666775782903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,4,128,1,float16,float16,0,0.02086399992307027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,4,128,1,float16,fp8,0,0.020842666427294414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,4,128,1,fp8,fp8,0,0.023141334454218548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,24,128,1,float16,float16,0,0.017008000363906223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,8,128,1,float16,fp8,0,0.02093333254257838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,8,128,1,fp8,fp8,0,0.024154665569464367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,24,8,128,1,float16,float16,0,0.021018666525681812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,1,128,1,float16,float16,0,0.017055999487638474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,1,128,1,float16,fp8,0,0.017765333255132038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,24,128,1,fp8,fp8,0,0.019573333362738293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,24,128,1,float16,fp8,0,0.020096000283956528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,1,128,1,fp8,fp8,0,0.018687999496857326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,2,128,1,float16,float16,0,0.017162666966517765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,4,128,1,float16,float16,0,0.017583999782800674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,2,128,1,fp8,fp8,0,0.01899733394384384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,2,128,1,float16,fp8,0,0.017664000391960144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,8,128,1,float16,float16,0,0.01740266631046931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,4,128,1,fp8,fp8,0,0.018922666708628338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,8,128,1,float16,fp8,0,0.017621333400408428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,8,128,1,fp8,fp8,0,0.019167999426523846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,24,128,1,float16,float16,0,0.016629333297411602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,24,4,128,1,float16,fp8,0,0.017562666287024815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,1,128,1,float16,float16,0,0.016970666746298473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,24,128,1,fp8,fp8,0,0.018624000251293182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,24,128,1,float16,fp8,0,0.01700266698996226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,1,128,1,float16,fp8,0,0.016949333250522614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,1,128,1,fp8,fp8,0,0.018533332894245785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,2,128,1,float16,float16,0,0.017029333859682083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,2,128,1,fp8,fp8,0,0.0184906671444575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,4,128,1,float16,float16,0,0.01693333312869072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,8,128,1,float16,float16,0,0.01659199967980385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,4,128,1,fp8,fp8,0,0.019066666563351948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,2,128,1,float16,fp8,0,0.016976000120242436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,4,128,1,float16,fp8,0,0.017029333859682083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,8,128,1,float16,fp8,0,0.017184000462293625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,24,8,128,1,fp8,fp8,0,0.018309333672126133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,24,128,1,float16,fp8,0,0.01684800038735072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,24,128,1,float16,float16,0,0.016421332955360413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,24,128,1,fp8,fp8,0,0.0184906671444575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,1,128,1,fp8,fp8,0,0.018218666315078735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,2,128,1,float16,float16,0,0.01653333380818367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,2,128,1,float16,fp8,0,0.016997333616018295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,1,128,1,float16,fp8,0,0.01717866708834966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,2,128,1,fp8,fp8,0,0.01820266619324684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,1,128,1,float16,float16,0,0.01653333380818367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,4,128,1,float16,float16,0,0.017045332739750545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,4,128,1,fp8,fp8,0,0.018522666146357853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,4,128,1,float16,fp8,0,0.016890666137139004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,8,128,1,float16,float16,0,0.01687466725707054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,8,128,1,fp8,fp8,0,0.018581333259741466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,24,8,128,1,float16,fp8,0,0.017125333348910015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,1,128,1,fp8,fp8,0,11.041098276774088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,1,128,1,float16,float16,0,16.823776245117188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,1,128,1,float16,fp8,0,16.631567637125652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,2,128,1,float16,fp8,0,16.73861312866211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,2,128,1,fp8,fp8,0,11.28396224975586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,2,128,1,float16,float16,0,16.688826243082683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,4,128,1,float16,float16,0,16.55404281616211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,4,128,1,fp8,fp8,0,10.965520222981771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,4,128,1,float16,fp8,0,16.76410675048828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,8,128,1,fp8,fp8,0,11.384698232014975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,8,128,1,float16,float16,0,16.85214360555013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,16,128,1,float16,float16,0,8.276592254638672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,16,8,128,1,float16,fp8,0,16.976229349772137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,16,128,1,float16,fp8,0,8.288864135742188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,1,128,1,float16,float16,0,8.405354817708334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,1,128,1,fp8,fp8,0,5.334463755289714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,1,128,1,float16,fp8,0,8.535018920898438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,2,128,1,fp8,fp8,0,5.429109573364258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,2,128,1,float16,fp8,0,8.429941177368164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,2,128,1,float16,float16,0,8.38378651936849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,16,128,1,fp8,fp8,0,5.549269358317058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,4,128,1,float16,float16,0,8.478288014729818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,4,128,1,fp8,fp8,0,5.634618759155273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,4,128,1,float16,fp8,0,8.483423868815104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,16,128,1,float16,float16,0,4.002426783243815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,1,128,1,float16,float16,0,4.065690676371257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,8,128,1,float16,float16,0,8.337125142415365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,8,128,1,fp8,fp8,0,5.449685414632161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,16,8,128,1,float16,fp8,0,8.472176233927408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,16,128,1,float16,fp8,0,4.193925221761067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,1,128,1,float16,fp8,0,4.11406930287679
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,16,128,1,fp8,fp8,0,2.8268534342447915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,1,128,1,fp8,fp8,0,2.762650807698568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,2,128,1,float16,float16,0,4.1665652592976885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,2,128,1,fp8,fp8,0,2.7662293116251626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,2,128,1,float16,fp8,0,3.9940694173177085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,4,128,1,float16,float16,0,4.1491092046101885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,4,128,1,float16,fp8,0,4.129541397094727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,4,128,1,fp8,fp8,0,2.783909479777018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,8,128,1,float16,float16,0,4.239952087402344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,8,128,1,fp8,fp8,0,2.795877456665039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,16,128,1,float16,float16,0,2.1073226928710938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,1,128,1,float16,float16,0,2.1714986165364585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,16,128,1,float16,fp8,0,2.103328069051107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,16,8,128,1,float16,fp8,0,4.176378568013509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,16,128,1,fp8,fp8,0,1.4909760157267253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,1,128,1,float16,fp8,0,2.1242453257242837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,1,128,1,fp8,fp8,0,1.4842400550842285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,2,128,1,float16,fp8,0,2.160319964090983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,2,128,1,fp8,fp8,0,1.4907946586608887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,4,128,1,float16,float16,0,2.1431360244750977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,4,128,1,float16,fp8,0,2.1272427241007485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,2,128,1,float16,float16,0,2.1373440424601235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,4,128,1,fp8,fp8,0,1.4972853660583496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,8,128,1,float16,float16,0,2.158906618754069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,8,128,1,float16,fp8,0,2.173818588256836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,16,8,128,1,fp8,fp8,0,1.5036959648132324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,1,128,1,float16,fp8,0,9.83183479309082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,1,128,1,float16,float16,0,9.613029479980469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,1,128,1,fp8,fp8,0,6.379909515380859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,2,128,1,float16,float16,0,9.650746663411459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,2,128,1,fp8,fp8,0,6.599280039469401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,2,128,1,float16,fp8,0,9.644543965657553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,4,128,1,float16,float16,0,9.778671900431315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,4,128,1,float16,fp8,0,9.768410364786783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,4,128,1,fp8,fp8,0,6.350255966186523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,16,128,1,float16,float16,0,4.671231905619304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,8,128,1,float16,float16,0,9.755114873250326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,16,128,1,float16,fp8,0,4.767098744710286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,1,128,1,float16,float16,0,4.584266662597656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,8,128,1,fp8,fp8,0,6.6867415110270185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,16,8,128,1,float16,fp8,0,9.733642578125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,1,128,1,float16,fp8,0,4.638261477152507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,1,128,1,fp8,fp8,0,3.184000015258789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,16,128,1,fp8,fp8,0,3.268159866333008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,2,128,1,float16,float16,0,4.601984024047852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,2,128,1,fp8,fp8,0,3.2018346786499023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,2,128,1,float16,fp8,0,4.66213321685791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,4,128,1,float16,float16,0,4.7303892771403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,4,128,1,float16,fp8,0,4.593167940775554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,4,128,1,fp8,fp8,0,3.2087891896565757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,16,128,1,float16,float16,0,2.367290655771891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,8,128,1,float16,float16,0,4.948357264200847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,8,128,1,float16,fp8,0,4.61893876393636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,16,8,128,1,fp8,fp8,0,3.250981330871582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,16,128,1,float16,fp8,0,2.368245283762614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,1,128,1,float16,float16,0,2.3758133252461753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,1,128,1,float16,fp8,0,2.3590505917867026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,16,128,1,fp8,fp8,0,1.693994681040446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,1,128,1,fp8,fp8,0,1.6675732930501301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,2,128,1,float16,float16,0,2.3842080434163413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,2,128,1,float16,fp8,0,2.4187520345052085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,2,128,1,fp8,fp8,0,1.6730772654215496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,4,128,1,float16,float16,0,2.3846774101257324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,4,128,1,fp8,fp8,0,1.6788160006205242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,4,128,1,float16,fp8,0,2.3698933919270835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,8,128,1,float16,float16,0,2.402496019999186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,16,128,1,float16,float16,0,1.2822720209757488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,8,128,1,float16,fp8,0,2.3949599266052246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,16,8,128,1,fp8,fp8,0,1.6979093551635742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,1,128,1,float16,float16,0,1.2881759802500408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,1,128,1,float16,fp8,0,1.278154691060384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,16,128,1,float16,fp8,0,1.2675200303395588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,16,128,1,fp8,fp8,0,0.8696959813435873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,1,128,1,fp8,fp8,0,0.858138640721639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,2,128,1,float16,float16,0,1.2936960061391194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,2,128,1,float16,fp8,0,1.2975040276845295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,2,128,1,fp8,fp8,0,0.8594506581624349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,4,128,1,float16,float16,0,1.294474681218465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,4,128,1,fp8,fp8,0,0.8629919687906901
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,4,128,1,float16,fp8,0,1.2852373123168945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,8,128,1,float16,float16,0,1.2997492949167888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,8,128,1,float16,fp8,0,1.2880213260650635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,16,8,128,1,fp8,fp8,0,0.875098705291748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,1,128,1,float16,float16,0,6.73747189839681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,1,128,1,float16,fp8,0,6.7236372629801435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,1,128,1,fp8,fp8,0,4.573466618855794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,2,128,1,float16,float16,0,6.831482569376628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,2,128,1,fp8,fp8,0,4.538991928100586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,2,128,1,float16,fp8,0,6.836314519246419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,4,128,1,float16,float16,0,6.968186696370442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,4,128,1,float16,fp8,0,6.736991882324219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,4,128,1,fp8,fp8,0,4.595957438151042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,8,128,1,float16,float16,0,6.980496088663737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,16,128,1,float16,float16,0,3.3075574239095054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,16,128,1,float16,fp8,0,3.342495918273926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,8,128,1,fp8,fp8,0,4.641909281412761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,1,128,1,float16,float16,0,3.2825921376546225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,16,8,128,1,float16,fp8,0,6.690144220987956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,1,128,1,fp8,fp8,0,2.313536008199056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,1,128,1,float16,fp8,0,3.34171199798584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,16,128,1,fp8,fp8,0,2.3685439427693686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,2,128,1,float16,float16,0,3.415445327758789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,2,128,1,float16,fp8,0,3.254314740498861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,2,128,1,fp8,fp8,0,2.3178933461507163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,4,128,1,float16,float16,0,3.286831855773926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,4,128,1,fp8,fp8,0,2.333087921142578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,4,128,1,float16,fp8,0,3.2729705174764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,16,128,1,float16,float16,0,1.7199254035949707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,8,128,1,float16,float16,0,3.3119147618611655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,8,128,1,float16,fp8,0,3.358112017313639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,16,8,128,1,fp8,fp8,0,2.365253289540609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,16,128,1,float16,fp8,0,1.699130694071452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,1,128,1,float16,float16,0,1.7213652928670247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,1,128,1,float16,fp8,0,1.7009600003560383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,1,128,1,fp8,fp8,0,1.2234293619791667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,16,128,1,fp8,fp8,0,1.249135971069336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,2,128,1,float16,fp8,0,1.7049493789672852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,2,128,1,float16,float16,0,1.7518720626831055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,2,128,1,fp8,fp8,0,1.2241120338439941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,4,128,1,float16,float16,0,1.731829325358073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,4,128,1,float16,fp8,0,1.709338665008545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,4,128,1,fp8,fp8,0,1.2343680063883464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,8,128,1,float16,float16,0,1.7442293167114258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,16,128,1,float16,float16,0,0.9356479644775391
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,8,128,1,float16,fp8,0,1.7229973475138347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,16,8,128,1,fp8,fp8,0,1.2456479867299397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,16,128,1,float16,fp8,0,0.9332746664683024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,1,128,1,float16,float16,0,0.9435839653015137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,16,128,1,fp8,fp8,0,0.644757350285848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,1,128,1,float16,fp8,0,0.9368267059326172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,2,128,1,float16,float16,0,0.9489759604136149
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,1,128,1,fp8,fp8,0,0.6311359802881876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,2,128,1,float16,fp8,0,0.9469119707743326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,2,128,1,fp8,fp8,0,0.6336319843928019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,4,128,1,float16,float16,0,0.9475200176239014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,4,128,1,float16,fp8,0,0.9387626647949219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,4,128,1,fp8,fp8,0,0.6371733347574869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,8,128,1,float16,float16,0,0.9523839950561523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,8,128,1,fp8,fp8,0,0.6465493440628052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,16,8,128,1,float16,fp8,0,0.9407467047373453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,1,128,1,float16,float16,0,9.057514826456705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,1,128,1,float16,fp8,0,8.739237467447916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,1,128,1,fp8,fp8,0,6.039690653483073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,2,128,1,fp8,fp8,0,6.179770787556966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,2,128,1,float16,fp8,0,9.014522552490234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,2,128,1,float16,float16,0,9.032277425130209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,4,128,1,float16,float16,0,9.11797841389974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,4,128,1,float16,fp8,0,9.13921038309733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,4,128,1,fp8,fp8,0,6.138970692952474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,16,128,1,float16,float16,0,4.370778719584147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,8,128,1,float16,float16,0,9.30197842915853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,8,128,1,fp8,fp8,0,6.277370452880859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,16,128,1,float16,fp8,0,4.455866813659668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,1,128,1,float16,float16,0,4.424362818400065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,16,8,128,1,float16,fp8,0,9.31114133199056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,1,128,1,fp8,fp8,0,3.0582027435302734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,2,128,1,float16,float16,0,4.293477376302083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,1,128,1,float16,fp8,0,4.474090576171875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,16,128,1,fp8,fp8,0,3.147712071736654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,2,128,1,fp8,fp8,0,3.065802574157715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,2,128,1,float16,fp8,0,4.465493202209473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,4,128,1,float16,float16,0,4.424341201782227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,4,128,1,float16,fp8,0,4.266202608744304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,4,128,1,fp8,fp8,0,3.0871359507242837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,16,128,1,float16,fp8,0,2.2471787134806314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,16,128,1,float16,float16,0,2.2090506553649902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,8,128,1,fp8,fp8,0,3.1261866887410483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,8,128,1,float16,fp8,0,4.532997449239095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,1,128,1,float16,float16,0,2.1933600107828775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,16,8,128,1,float16,float16,0,4.443941434224446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,1,128,1,float16,fp8,0,2.208794593811035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,16,128,1,fp8,fp8,0,1.6303733189900715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,1,128,1,fp8,fp8,0,1.576149304707845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,2,128,1,float16,float16,0,2.1977866490681968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,2,128,1,float16,fp8,0,2.1782453854878745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,2,128,1,fp8,fp8,0,1.5842399597167969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,4,128,1,float16,float16,0,2.215989271799723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,4,128,1,float16,fp8,0,2.2225653330485025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,4,128,1,fp8,fp8,0,1.592741330464681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,16,128,1,float16,float16,0,1.167690674463908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,8,128,1,float16,fp8,0,2.1977920532226562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,8,128,1,fp8,fp8,0,1.6194613774617512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,16,8,128,1,float16,float16,0,2.217690626780192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,1,128,1,float16,float16,0,1.1650880177815754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,1,128,1,float16,fp8,0,1.1501333713531494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,16,128,1,float16,fp8,0,1.164511998494466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,1,128,1,fp8,fp8,0,0.8466453552246094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,16,128,1,fp8,fp8,0,0.8697546323140463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,2,128,1,float16,float16,0,1.168346643447876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,2,128,1,fp8,fp8,0,0.8480052947998047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,4,128,1,float16,float16,0,1.1700692971547444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,2,128,1,float16,fp8,0,1.1517546971638997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,4,128,1,fp8,fp8,0,0.8521280288696289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,8,128,1,float16,float16,0,1.1942346890767415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,8,128,1,float16,fp8,0,1.1667253176371257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,4,128,1,float16,fp8,0,1.1605707009633381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,16,128,1,float16,float16,0,0.6509546836217245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,16,8,128,1,fp8,fp8,0,0.8648906548817953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,16,128,1,float16,fp8,0,0.644976019859314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,1,128,1,float16,float16,0,0.6565973361333212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,1,128,1,float16,fp8,0,0.6462986469268799
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,16,128,1,fp8,fp8,0,0.4529920021692912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,1,128,1,fp8,fp8,0,0.4413173198699951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,2,128,1,float16,float16,0,0.6571786801020304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,2,128,1,float16,fp8,0,0.6483039855957031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,4,128,1,float16,float16,0,0.6607733170191447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,4,128,1,float16,fp8,0,0.6507946650187174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,4,128,1,fp8,fp8,0,0.44582398732503253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,2,128,1,fp8,fp8,0,0.44234665234883624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,8,128,1,float16,float16,0,0.661733349164327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,8,128,1,float16,fp8,0,0.6545706590016683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,16,8,128,1,fp8,fp8,0,0.45127999782562256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,1,128,1,fp8,fp8,0,3.683237393697103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,1,128,1,float16,fp8,0,5.021231969197591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,1,128,1,float16,float16,0,5.256202697753906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,2,128,1,fp8,fp8,0,3.7018187840779624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,2,128,1,float16,fp8,0,5.275525410970052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,2,128,1,float16,float16,0,5.052645365397136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,4,128,1,float16,float16,0,5.346127827962239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,4,128,1,float16,fp8,0,5.113679885864258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,4,128,1,fp8,fp8,0,3.7280267079671225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,16,128,1,float16,float16,0,2.6314719518025718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,16,128,1,float16,fp8,0,2.5947839419047036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,8,128,1,float16,float16,0,5.26969051361084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,1,128,1,float16,float16,0,2.5674079259236655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,8,128,1,float16,fp8,0,5.2204640706380205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,16,8,128,1,fp8,fp8,0,3.786479949951172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,1,128,1,float16,fp8,0,2.520965258280436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,1,128,1,fp8,fp8,0,1.882863998413086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,2,128,1,float16,float16,0,2.615429401397705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,2,128,1,float16,fp8,0,2.539776007334391
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,16,128,1,fp8,fp8,0,1.9620532989501953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,2,128,1,fp8,fp8,0,1.8923412958780925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,4,128,1,float16,float16,0,2.612720012664795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,4,128,1,fp8,fp8,0,1.9073333740234375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,4,128,1,float16,fp8,0,2.5453920364379883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,16,128,1,float16,float16,0,1.3530186017354329
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,8,128,1,float16,float16,0,2.6562827428181968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,8,128,1,fp8,fp8,0,1.9385493596394856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,16,8,128,1,float16,fp8,0,2.609440008799235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,1,128,1,float16,float16,0,1.3274292945861816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,1,128,1,float16,fp8,0,1.3095093568166096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,16,128,1,fp8,fp8,0,1.027781327565511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,16,128,1,float16,fp8,0,1.3462400436401367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,1,128,1,fp8,fp8,0,0.9814133644104004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,2,128,1,float16,float16,0,1.3296480178833008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,2,128,1,float16,fp8,0,1.3148213227589924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,2,128,1,fp8,fp8,0,0.986186663309733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,4,128,1,float16,float16,0,1.337007999420166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,4,128,1,float16,fp8,0,1.3198506832122803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,4,128,1,fp8,fp8,0,0.9948000113169352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,8,128,1,float16,float16,0,1.37009064356486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,16,128,1,float16,float16,0,0.7220053672790527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,8,128,1,float16,fp8,0,1.3329386711120605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,16,8,128,1,fp8,fp8,0,1.0108373165130615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,1,128,1,float16,float16,0,0.7200373013814291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,16,128,1,float16,fp8,0,0.7169866561889648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,16,128,1,fp8,fp8,0,0.5318719943364462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,1,128,1,float16,fp8,0,0.7128213246663412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,2,128,1,float16,fp8,0,0.7119946479797363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,2,128,1,float16,float16,0,0.7194773356119791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,1,128,1,fp8,fp8,0,0.5059039990107218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,2,128,1,fp8,fp8,0,0.5079946517944336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,4,128,1,float16,float16,0,0.7239840030670166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,4,128,1,float16,fp8,0,0.7147680123647054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,4,128,1,fp8,fp8,0,0.5134026606877645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,8,128,1,float16,fp8,0,0.7187306880950928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,8,128,1,float16,float16,0,0.72980268796285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,16,8,128,1,fp8,fp8,0,0.523690660794576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,16,128,1,float16,float16,0,0.3712799946467082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,1,128,1,float16,float16,0,0.36344532171885174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,16,128,1,float16,fp8,0,0.36586666107177734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,16,128,1,fp8,fp8,0,0.2938080032666524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,1,128,1,float16,fp8,0,0.35677866141001385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,1,128,1,fp8,fp8,0,0.2849386731783549
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,2,128,1,float16,float16,0,0.3644213279088338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,2,128,1,fp8,fp8,0,0.28594666719436646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,4,128,1,float16,fp8,0,0.36002667744954425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,2,128,1,float16,fp8,0,0.3583413362503052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,4,128,1,float16,float16,0,0.3669653336207072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,4,128,1,fp8,fp8,0,0.2874559958775838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,8,128,1,float16,float16,0,0.3710346619288127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,8,128,1,float16,fp8,0,0.3641546567281087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,16,8,128,1,fp8,fp8,0,0.2927306691805522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,1,128,1,float16,float16,0,5.123594601949056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,1,128,1,fp8,fp8,0,3.7788852055867515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,1,128,1,float16,fp8,0,5.006634712219238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,2,128,1,float16,fp8,0,5.072426795959473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,2,128,1,float16,float16,0,5.166154543558757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,2,128,1,fp8,fp8,0,3.7970399856567383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,4,128,1,float16,float16,0,5.166693369547526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,4,128,1,float16,fp8,0,4.996533393859863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,4,128,1,fp8,fp8,0,3.8363625208536782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,16,128,1,float16,float16,0,2.6071252822875977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,16,128,1,float16,fp8,0,2.6330080032348633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,8,128,1,float16,float16,0,5.340789159138997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,1,128,1,float16,float16,0,2.5057759284973145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,8,128,1,float16,fp8,0,5.177440007527669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,16,8,128,1,fp8,fp8,0,3.9155734380086265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,1,128,1,float16,fp8,0,2.4585280418395996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,1,128,1,fp8,fp8,0,1.9169813791910808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,2,128,1,float16,float16,0,2.524522622426351
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,2,128,1,fp8,fp8,0,1.9255626996358235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,16,128,1,fp8,fp8,0,2.0291520754496255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,2,128,1,float16,fp8,0,2.4710826873779297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,4,128,1,float16,float16,0,2.5370987256368003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,4,128,1,float16,fp8,0,2.4854346911112466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,4,128,1,fp8,fp8,0,1.9454560279846191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,16,128,1,float16,float16,0,1.3336480458577473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,16,128,1,float16,fp8,0,1.3101119995117188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,8,128,1,float16,float16,0,2.5865012804667153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,8,128,1,fp8,fp8,0,1.9873119990030925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,16,8,128,1,float16,fp8,0,2.5365333557128906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,1,128,1,float16,float16,0,1.291530688603719
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,1,128,1,float16,fp8,0,1.2557600339253743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,1,128,1,fp8,fp8,0,0.9819680054982504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,2,128,1,float16,float16,0,1.285919984181722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,2,128,1,float16,fp8,0,1.2610186735788982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,2,128,1,fp8,fp8,0,0.9882400035858154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,16,128,1,fp8,fp8,0,1.047818660736084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,4,128,1,float16,float16,0,1.306058645248413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,4,128,1,float16,fp8,0,1.268730640411377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,4,128,1,fp8,fp8,0,1.0000053246815999
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,8,128,1,float16,fp8,0,1.2875839869181316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,8,128,1,float16,float16,0,1.3149173259735107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,16,128,1,float16,fp8,0,0.697322686513265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,16,128,1,float16,float16,0,0.6981333096822103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,16,8,128,1,fp8,fp8,0,1.0222400029500325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,1,128,1,float16,float16,0,0.6766613324483236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,16,128,1,fp8,fp8,0,0.5602293411890665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,1,128,1,float16,fp8,0,0.6658826669057211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,1,128,1,fp8,fp8,0,0.5253386497497559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,2,128,1,float16,float16,0,0.6849919954935709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,2,128,1,fp8,fp8,0,0.5282293160756429
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,4,128,1,float16,float16,0,0.68504532178243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,2,128,1,float16,fp8,0,0.6698079903920492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,4,128,1,float16,fp8,0,0.6742613315582275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,4,128,1,fp8,fp8,0,0.5335679848988851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,8,128,1,float16,float16,0,0.694271961847941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,16,128,1,float16,float16,0,0.38598934809366864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,8,128,1,fp8,fp8,0,0.5472586552302042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,16,8,128,1,float16,fp8,0,0.6879093647003174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,1,128,1,float16,float16,0,0.37907731533050537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,16,128,1,fp8,fp8,0,0.29629333813985187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,16,128,1,float16,fp8,0,0.3810880184173584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,1,128,1,float16,fp8,0,0.3723893165588379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,1,128,1,fp8,fp8,0,0.27611732482910156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,2,128,1,float16,float16,0,0.3808853228886922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,2,128,1,fp8,fp8,0,0.27646400531133014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,2,128,1,float16,fp8,0,0.37565334637959796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,4,128,1,float16,fp8,0,0.377621332804362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,4,128,1,float16,float16,0,0.3841013511021932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,4,128,1,fp8,fp8,0,0.2805973291397095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,16,128,1,float16,float16,0,0.20071999231974283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,8,128,1,fp8,fp8,0,0.286463995774587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,8,128,1,float16,fp8,0,0.38180267810821533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,16,128,1,float16,fp8,0,0.1967359979947408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,16,128,1,fp8,fp8,0,0.1694773236910502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,16,8,128,1,float16,float16,0,0.38718398412068683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,1,128,1,float16,float16,0,0.1934559941291809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,1,128,1,float16,fp8,0,0.19089066982269287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,1,128,1,fp8,fp8,0,0.15934399763743082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,2,128,1,float16,fp8,0,0.19048533837000528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,2,128,1,fp8,fp8,0,0.1601706643899282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,2,128,1,float16,float16,0,0.19428799549738565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,4,128,1,float16,float16,0,0.19526400168736777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,4,128,1,fp8,fp8,0,0.1629759967327118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,4,128,1,float16,fp8,0,0.19233065843582153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,8,128,1,float16,fp8,0,0.194815993309021
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,8,128,1,float16,float16,0,0.1981173356374105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,16,8,128,1,fp8,fp8,0,0.1665013333161672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,1,128,1,float16,fp8,0,3.0135574340820312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,1,128,1,float16,float16,0,3.1187305450439453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,1,128,1,fp8,fp8,0,2.4269280433654785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,2,128,1,float16,fp8,0,3.0262667338053384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,2,128,1,fp8,fp8,0,2.439098676045736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,2,128,1,float16,float16,0,3.1144641240437827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,4,128,1,float16,float16,0,3.1310240427652993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,4,128,1,fp8,fp8,0,2.477274735768636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,4,128,1,float16,fp8,0,3.0574560165405273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,16,128,1,float16,float16,0,1.6576213836669922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,16,128,1,float16,fp8,0,1.6162187258402507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,8,128,1,float16,float16,0,3.258000055948893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,8,128,1,fp8,fp8,0,2.530794620513916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,1,128,1,float16,float16,0,1.5545760790507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,1,128,1,float16,fp8,0,1.5207200050354004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,1,128,1,fp8,fp8,0,1.2354133129119873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,16,8,128,1,float16,fp8,0,3.1288426717122397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,2,128,1,float16,float16,0,1.576101303100586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,2,128,1,float16,fp8,0,1.5246772766113281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,2,128,1,fp8,fp8,0,1.2403093179066975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,16,128,1,fp8,fp8,0,1.3280426661173503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,4,128,1,float16,float16,0,1.5774720509847004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,4,128,1,float16,fp8,0,1.539962609608968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,4,128,1,fp8,fp8,0,1.255674680074056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,8,128,1,float16,float16,0,1.624773343404134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,16,128,1,float16,float16,0,0.8487359682718912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,8,128,1,fp8,fp8,0,1.2888800303141277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,16,8,128,1,float16,fp8,0,1.569061279296875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,16,128,1,float16,fp8,0,0.8354079723358154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,16,128,1,fp8,fp8,0,0.6932373046875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,1,128,1,float16,float16,0,0.8036533196767172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,1,128,1,float16,fp8,0,0.7913333574930826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,1,128,1,fp8,fp8,0,0.6387626727422079
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,2,128,1,float16,float16,0,0.8051786422729492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,2,128,1,float16,fp8,0,0.7901653448740641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,2,128,1,fp8,fp8,0,0.6431413491566976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,4,128,1,float16,float16,0,0.8136053085327148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,4,128,1,float16,fp8,0,0.7969600359598795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,4,128,1,fp8,fp8,0,0.6530239979426066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,8,128,1,float16,float16,0,0.8256959915161133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,16,128,1,float16,float16,0,0.45230400562286377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,8,128,1,float16,fp8,0,0.8103360335032145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,16,8,128,1,fp8,fp8,0,0.667189359664917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,16,128,1,float16,fp8,0,0.44369598229726154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,1,128,1,fp8,fp8,0,0.3314719994862874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,1,128,1,float16,fp8,0,0.4241866668065389
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,1,128,1,float16,float16,0,0.43332799275716144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,2,128,1,float16,float16,0,0.43510401248931885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,2,128,1,float16,fp8,0,0.4261920054753621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,16,128,1,fp8,fp8,0,0.3650826613108317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,2,128,1,fp8,fp8,0,0.33429332574208576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,4,128,1,float16,fp8,0,0.43026665846506756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,4,128,1,float16,float16,0,0.4391253391901652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,4,128,1,fp8,fp8,0,0.3390666643778483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,8,128,1,float16,fp8,0,0.43638932704925537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,8,128,1,float16,float16,0,0.4447520176569621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,16,128,1,float16,float16,0,0.23694399992624918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,1,128,1,float16,float16,0,0.2232159972190857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,1,128,1,float16,fp8,0,0.21779733896255493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,16,8,128,1,fp8,fp8,0,0.34975465138753253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,1,128,1,fp8,fp8,0,0.18528532981872559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,16,128,1,float16,fp8,0,0.23385600248972574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,16,128,1,fp8,fp8,0,0.20077866315841675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,2,128,1,float16,float16,0,0.22366933027903238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,2,128,1,fp8,fp8,0,0.18618667125701904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,2,128,1,float16,fp8,0,0.2185653249422709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,4,128,1,float16,fp8,0,0.22004799048105875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,4,128,1,fp8,fp8,0,0.1886133352915446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,4,128,1,float16,float16,0,0.22501333554585776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,8,128,1,float16,float16,0,0.23110399643580118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,8,128,1,float16,fp8,0,0.2261013388633728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,16,8,128,1,fp8,fp8,0,0.19313599665959677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,16,128,1,float16,float16,0,0.13838932911554971
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,16,128,1,float16,fp8,0,0.1356106698513031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,1,128,1,float16,float16,0,0.13084266583124796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,16,128,1,fp8,fp8,0,0.11974933743476868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,1,128,1,fp8,fp8,0,0.10967999696731567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,2,128,1,float16,float16,0,0.13145066301027933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,1,128,1,float16,fp8,0,0.12946666280428568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,2,128,1,float16,fp8,0,0.13057066996892294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,2,128,1,fp8,fp8,0,0.10995733737945557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,4,128,1,float16,float16,0,0.1334773302078247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,4,128,1,float16,fp8,0,0.1306719978650411
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,4,128,1,fp8,fp8,0,0.11432533462842305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,8,128,1,float16,float16,0,0.13520532846450806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,8,128,1,fp8,fp8,0,0.116976002852122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,16,8,128,1,float16,fp8,0,0.13370133439699808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,1,128,1,float16,float16,0,3.2878293991088867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,1,128,1,fp8,fp8,0,2.714474678039551
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,2,128,1,float16,float16,0,3.3175360361735025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,2,128,1,float16,fp8,0,3.175482749938965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,1,128,1,float16,fp8,0,3.1509758631388345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,2,128,1,fp8,fp8,0,2.720442771911621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,4,128,1,float16,float16,0,3.354991912841797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,4,128,1,fp8,fp8,0,2.7477814356486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,4,128,1,float16,fp8,0,3.2064746220906577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,16,128,1,float16,float16,0,1.8062987327575684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,16,128,1,float16,fp8,0,1.7211947441101074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,8,128,1,float16,float16,0,3.4630772272745767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,8,128,1,float16,fp8,0,3.2940746943155923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,1,128,1,float16,float16,0,1.638912041982015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,16,8,128,1,fp8,fp8,0,2.830256144205729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,1,128,1,float16,fp8,0,1.587231953938802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,1,128,1,fp8,fp8,0,1.3530720074971516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,2,128,1,float16,float16,0,1.6509920756022136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,2,128,1,float16,fp8,0,1.5859413146972656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,2,128,1,fp8,fp8,0,1.360655943552653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,16,128,1,fp8,fp8,0,1.482474644978841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,4,128,1,float16,float16,0,1.6628586451212566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,4,128,1,float16,fp8,0,1.6189653078715007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,4,128,1,fp8,fp8,0,1.3813707033793132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,16,128,1,float16,float16,0,0.8979466756184896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,8,128,1,float16,float16,0,1.7199519475301106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,8,128,1,float16,fp8,0,1.6450400352478027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,16,8,128,1,fp8,fp8,0,1.4230027198791504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,1,128,1,float16,float16,0,0.8241013685862223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,16,128,1,float16,fp8,0,0.8733812967936198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,16,128,1,fp8,fp8,0,0.7600213686625162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,1,128,1,float16,fp8,0,0.8016586303710938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,1,128,1,fp8,fp8,0,0.6881493727366129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,2,128,1,float16,float16,0,0.8276267051696777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,2,128,1,float16,fp8,0,0.8060373465220133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,2,128,1,fp8,fp8,0,0.6929546991984049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,4,128,1,float16,fp8,0,0.8185226917266846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,4,128,1,fp8,fp8,0,0.7056639989217123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,4,128,1,float16,float16,0,0.836629311243693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,8,128,1,float16,float16,0,0.8563199838002523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,8,128,1,float16,fp8,0,0.8347093264261881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,16,128,1,float16,float16,0,0.4694720109303792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,16,8,128,1,fp8,fp8,0,0.7274186611175537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,1,128,1,float16,float16,0,0.43456534544626874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,16,128,1,float16,fp8,0,0.46118934949239093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,1,128,1,fp8,fp8,0,0.3651946783065796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,1,128,1,float16,fp8,0,0.4245866537094116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,16,128,1,fp8,fp8,0,0.40241066614786786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,2,128,1,float16,float16,0,0.44015999635060626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,2,128,1,float16,fp8,0,0.4270453453063965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,2,128,1,fp8,fp8,0,0.36748266220092773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,4,128,1,float16,fp8,0,0.43095465501149494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,4,128,1,float16,float16,0,0.4432160059611003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,4,128,1,fp8,fp8,0,0.37407998243967694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,8,128,1,float16,float16,0,0.45148801803588867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,8,128,1,float16,fp8,0,0.4432426691055298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,16,8,128,1,fp8,fp8,0,0.38412264982859295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,1,128,1,float16,float16,0,0.24059200286865234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,1,128,1,float16,fp8,0,0.2348639965057373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,16,128,1,float16,fp8,0,0.2505706747372945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,1,128,1,fp8,fp8,0,0.19210666418075562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,16,128,1,float16,float16,0,0.2552746733029683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,2,128,1,float16,float16,0,0.2418986757596334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,16,128,1,fp8,fp8,0,0.21606934070587158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,2,128,1,float16,fp8,0,0.23805334170659384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,2,128,1,fp8,fp8,0,0.19368000825246176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,4,128,1,float16,float16,0,0.24494399627049765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,4,128,1,float16,fp8,0,0.24065067370732626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,8,128,1,float16,float16,0,0.2493706742922465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,8,128,1,float16,fp8,0,0.24623999993006387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,4,128,1,fp8,fp8,0,0.19638399283091226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,16,8,128,1,fp8,fp8,0,0.20426666736602783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,16,128,1,float16,float16,0,0.1362986663977305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,16,128,1,float16,fp8,0,0.13336533308029175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,1,128,1,float16,float16,0,0.12593600153923035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,1,128,1,float16,fp8,0,0.12211199601491292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,16,128,1,fp8,fp8,0,0.1225226620833079
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,1,128,1,fp8,fp8,0,0.10993599891662598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,2,128,1,float16,float16,0,0.1269653340180715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,2,128,1,float16,fp8,0,0.12397866447766621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,2,128,1,fp8,fp8,0,0.11202667156855266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,4,128,1,float16,float16,0,0.1279253363609314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,4,128,1,float16,fp8,0,0.12544533610343933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,4,128,1,fp8,fp8,0,0.11458667119344075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,8,128,1,float16,float16,0,0.1304266651471456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,8,128,1,float16,fp8,0,0.12798933188120523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,16,8,128,1,fp8,fp8,0,0.11772267023722331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,16,128,1,float16,float16,0,0.08018666505813599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,16,128,1,float16,fp8,0,0.08003733555475871
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,16,128,1,fp8,fp8,0,0.07645333309968312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,1,128,1,float16,float16,0,0.0759200006723404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,1,128,1,fp8,fp8,0,0.06748266518115997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,1,128,1,float16,fp8,0,0.07482133309046428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,2,128,1,float16,fp8,0,0.07508799930413564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,2,128,1,fp8,fp8,0,0.067930668592453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,2,128,1,float16,float16,0,0.07645333309968312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,4,128,1,float16,float16,0,0.07645333309968312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,4,128,1,float16,fp8,0,0.07572266459465027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,4,128,1,fp8,fp8,0,0.0684799998998642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,8,128,1,float16,float16,0,0.07733333110809326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,8,128,1,float16,fp8,0,0.07718400160471599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,16,8,128,1,fp8,fp8,0,0.07179733117421468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,1,128,1,float16,float16,0,2.1565492947896323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,1,128,1,float16,fp8,0,2.034735997517904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,1,128,1,fp8,fp8,0,1.8323733011881511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,2,128,1,float16,float16,0,2.1774293581644693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,2,128,1,float16,fp8,0,2.0318613052368164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,2,128,1,fp8,fp8,0,1.8450826009114583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,4,128,1,float16,float16,0,2.1992692947387695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,4,128,1,float16,fp8,0,2.0634560585021973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,4,128,1,fp8,fp8,0,1.8719520568847656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,16,128,1,float16,float16,0,1.191269318262736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,16,128,1,float16,fp8,0,1.1286133130391438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,8,128,1,float16,float16,0,2.281205336252848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,8,128,1,float16,fp8,0,2.115664005279541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,1,128,1,float16,float16,0,1.0699840386708577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,16,8,128,1,fp8,fp8,0,1.9214879671732585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,1,128,1,float16,fp8,0,1.018181324005127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,1,128,1,fp8,fp8,0,0.9163466294606527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,2,128,1,float16,float16,0,1.0801653067270915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,2,128,1,float16,fp8,0,1.0252799987792969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,2,128,1,fp8,fp8,0,0.923904021581014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,16,128,1,fp8,fp8,0,1.018671989440918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,4,128,1,float16,fp8,0,1.0356160004933674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,4,128,1,float16,float16,0,1.0903786818186443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,4,128,1,fp8,fp8,0,0.9370079835255941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,16,128,1,float16,float16,0,0.6018720070521036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,8,128,1,float16,float16,0,1.1351946989695232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,8,128,1,float16,fp8,0,1.065178632736206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,16,128,1,float16,fp8,0,0.5823626518249512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,1,128,1,float16,float16,0,0.542954683303833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,16,8,128,1,fp8,fp8,0,0.9690240224202474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,16,128,1,fp8,fp8,0,0.5269973278045654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,1,128,1,float16,fp8,0,0.526586651802063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,1,128,1,fp8,fp8,0,0.47018134593963623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,2,128,1,float16,float16,0,0.5467679897944132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,2,128,1,float16,fp8,0,0.5287306706110636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,2,128,1,fp8,fp8,0,0.4748266537984212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,4,128,1,float16,float16,0,0.5528853336970011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,4,128,1,fp8,fp8,0,0.4819626808166504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,4,128,1,float16,fp8,0,0.5353333155314127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,8,128,1,float16,float16,0,0.5697493155797323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,16,128,1,float16,float16,0,0.3195306658744812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,8,128,1,float16,fp8,0,0.549946665763855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,16,8,128,1,fp8,fp8,0,0.5002346833546957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,16,128,1,float16,fp8,0,0.31116267045338947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,1,128,1,float16,float16,0,0.2913279930750529
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,1,128,1,float16,fp8,0,0.28381866216659546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,1,128,1,fp8,fp8,0,0.24604799350102743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,2,128,1,float16,float16,0,0.2939786712328593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,2,128,1,float16,fp8,0,0.2857653299967448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,16,128,1,fp8,fp8,0,0.28893333673477173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,2,128,1,fp8,fp8,0,0.2469866673151652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,4,128,1,float16,float16,0,0.29717334111531574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,4,128,1,float16,fp8,0,0.28965866565704346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,4,128,1,fp8,fp8,0,0.25244800249735516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,8,128,1,float16,float16,0,0.30453334252039593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,16,128,1,float16,float16,0,0.17821866273880005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,8,128,1,fp8,fp8,0,0.26360533634821576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,16,8,128,1,float16,fp8,0,0.2964640061060588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,1,128,1,float16,float16,0,0.15340800086657205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,1,128,1,float16,fp8,0,0.1498186687628428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,1,128,1,fp8,fp8,0,0.13586666186650595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,2,128,1,float16,float16,0,0.15465066830317178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,16,128,1,fp8,fp8,0,0.15228266517321268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,16,128,1,float16,fp8,0,0.17386666933695474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,2,128,1,float16,fp8,0,0.1507146656513214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,4,128,1,float16,float16,0,0.1569973329703013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,2,128,1,fp8,fp8,0,0.13768000404040018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,4,128,1,float16,fp8,0,0.1529973347981771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,4,128,1,fp8,fp8,0,0.13988266388575235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,8,128,1,float16,float16,0,0.16331733266512552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,8,128,1,float16,fp8,0,0.15916799505551657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,16,8,128,1,fp8,fp8,0,0.14453867077827454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,16,128,1,float16,float16,0,0.09612799684206645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,1,128,1,float16,float16,0,0.08686932921409607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,16,128,1,float16,fp8,0,0.09384533762931824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,16,128,1,fp8,fp8,0,0.09078400333722432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,1,128,1,float16,fp8,0,0.08549333612124126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,1,128,1,fp8,fp8,0,0.07765866816043854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,2,128,1,float16,fp8,0,0.08611200253168742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,2,128,1,fp8,fp8,0,0.08028799792130788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,4,128,1,float16,float16,0,0.08916800220807393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,4,128,1,float16,fp8,0,0.08859733740488689
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,2,128,1,float16,float16,0,0.08729066451390584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,4,128,1,fp8,fp8,0,0.08317333459854126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,8,128,1,float16,fp8,0,0.09010666608810425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,8,128,1,float16,float16,0,0.09103999535242717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,16,8,128,1,fp8,fp8,0,0.08700266480445862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,16,128,1,float16,float16,0,0.05983466903368632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,16,128,1,fp8,fp8,0,0.05840533475081126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,1,128,1,float16,float16,0,0.05686399837334951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,16,128,1,float16,fp8,0,0.058975999553998314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,1,128,1,float16,fp8,0,0.05619200070699056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,2,128,1,float16,float16,0,0.056927998860677086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,1,128,1,fp8,fp8,0,0.051541333397229515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,2,128,1,float16,fp8,0,0.05669333537419637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,2,128,1,fp8,fp8,0,0.05197866757710775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,4,128,1,float16,float16,0,0.056736002365748085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,4,128,1,fp8,fp8,0,0.052784000833829246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,4,128,1,float16,fp8,0,0.056202664971351624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,8,128,1,float16,float16,0,0.057071998715400696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,8,128,1,float16,fp8,0,0.05676800012588501
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,16,8,128,1,fp8,fp8,0,0.05397333204746246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,1,128,1,float16,float16,0,1.946239948272705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,1,128,1,fp8,fp8,0,1.7689812978108723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,1,128,1,float16,fp8,0,1.9463626543680828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,2,128,1,float16,fp8,0,1.9748533566792805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,2,128,1,float16,float16,0,1.9669493039449055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,2,128,1,fp8,fp8,0,1.8603253364562988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,4,128,1,float16,float16,0,2.0730932553609214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,4,128,1,float16,fp8,0,2.0954880714416504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,4,128,1,fp8,fp8,0,2.009760061899821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,16,128,1,float16,float16,0,1.264346679051717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,16,128,1,float16,fp8,0,1.2221813201904297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,8,128,1,float16,float16,0,2.2077813148498535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,8,128,1,float16,fp8,0,2.1853440602620444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,1,128,1,float16,float16,0,0.9886559645334879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,16,8,128,1,fp8,fp8,0,2.0327146848042807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,1,128,1,fp8,fp8,0,0.8774773279825846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,1,128,1,float16,fp8,0,0.9888052940368652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,2,128,1,float16,fp8,0,0.9990346431732178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,2,128,1,float16,float16,0,1.000069300333659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,2,128,1,fp8,fp8,0,0.938922643661499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,16,128,1,fp8,fp8,0,1.088912010192871
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,4,128,1,float16,float16,0,1.045914649963379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,4,128,1,float16,fp8,0,1.0394666989644368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,4,128,1,fp8,fp8,0,1.0120213031768799
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,8,128,1,float16,float16,0,1.100805362065633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,16,128,1,float16,float16,0,0.6453119913736979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,8,128,1,float16,fp8,0,1.0868266423543294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,16,128,1,float16,fp8,0,0.6190186738967896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,16,8,128,1,fp8,fp8,0,1.030618667602539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,1,128,1,float16,float16,0,0.509610652923584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,1,128,1,float16,fp8,0,0.5102986494700114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,16,128,1,fp8,fp8,0,0.554314653078715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,1,128,1,fp8,fp8,0,0.4500480095545451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,2,128,1,float16,float16,0,0.5159093141555786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,2,128,1,float16,fp8,0,0.5139893293380737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,2,128,1,fp8,fp8,0,0.47939733664194745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,4,128,1,float16,fp8,0,0.5245653390884399
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,4,128,1,float16,float16,0,0.5341813166936239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,16,128,1,float16,float16,0,0.3335466782251994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,4,128,1,fp8,fp8,0,0.5182079871495565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,8,128,1,fp8,fp8,0,0.5253546635309855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,8,128,1,float16,fp8,0,0.5538186629613241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,16,8,128,1,float16,float16,0,0.5617333253224691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,1,128,1,float16,float16,0,0.2693493366241455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,1,128,1,float16,fp8,0,0.2698826591173808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,16,128,1,float16,fp8,0,0.3229599992434184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,1,128,1,fp8,fp8,0,0.2392746607462565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,2,128,1,float16,float16,0,0.2736213405927022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,16,128,1,fp8,fp8,0,0.28852800528208417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,2,128,1,float16,fp8,0,0.27081600824991864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,2,128,1,fp8,fp8,0,0.24870934089024863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,4,128,1,float16,float16,0,0.2802186608314514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,4,128,1,float16,fp8,0,0.2784319917360942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,4,128,1,fp8,fp8,0,0.2693013350168864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,8,128,1,float16,float16,0,0.2967466711997986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,8,128,1,float16,fp8,0,0.29160000880559284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,16,128,1,float16,float16,0,0.1795253356297811
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,1,128,1,float16,float16,0,0.15026666720708212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,16,8,128,1,fp8,fp8,0,0.2717546621958415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,1,128,1,float16,fp8,0,0.14936533570289612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,16,128,1,fp8,fp8,0,0.1520853340625763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,1,128,1,fp8,fp8,0,0.12598400314648947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,16,128,1,float16,fp8,0,0.17546667655309042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,2,128,1,float16,fp8,0,0.1506613294283549
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,2,128,1,float16,float16,0,0.15189866224924722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,2,128,1,fp8,fp8,0,0.12787200013796488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,4,128,1,float16,float16,0,0.1567626694838206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,4,128,1,float16,fp8,0,0.1552906632423401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,8,128,1,float16,float16,0,0.16290133198102316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,4,128,1,fp8,fp8,0,0.1381493310133616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,8,128,1,float16,fp8,0,0.1604266663392385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,16,8,128,1,fp8,fp8,0,0.14019733667373657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,16,128,1,float16,fp8,0,0.09624532858530681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,16,128,1,float16,float16,0,0.09980799754460652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,1,128,1,float16,float16,0,0.07932266592979431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,16,128,1,fp8,fp8,0,0.0844586690266927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,1,128,1,fp8,fp8,0,0.06955733398596446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,1,128,1,float16,fp8,0,0.08004266520341237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,2,128,1,float16,float16,0,0.08066133161385854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,2,128,1,float16,fp8,0,0.0807360013326009
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,2,128,1,fp8,fp8,0,0.07172266642252605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,4,128,1,float16,fp8,0,0.08403199911117554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,4,128,1,float16,float16,0,0.08396800359090169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,4,128,1,fp8,fp8,0,0.0763626645008723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,8,128,1,float16,fp8,0,0.08549867073694865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,16,128,1,float16,float16,0,0.053120002150535583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,16,128,1,float16,fp8,0,0.05285866558551788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,16,128,1,fp8,fp8,0,0.04983466863632202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,1,128,1,float16,float16,0,0.046053335070610046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,8,128,1,fp8,fp8,0,0.07708799839019775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,16,8,128,1,float16,float16,0,0.08589333295822144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,1,128,1,float16,fp8,0,0.04641066491603851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,1,128,1,fp8,fp8,0,0.04196266829967499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,2,128,1,fp8,fp8,0,0.042693331837654114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,2,128,1,float16,fp8,0,0.046623999873797096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,2,128,1,float16,float16,0,0.04677866895993551
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,4,128,1,float16,float16,0,0.04832000037034353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,4,128,1,float16,fp8,0,0.047824000318845115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,4,128,1,fp8,fp8,0,0.04484800000985464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,8,128,1,float16,fp8,0,0.048997332652409874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,8,128,1,float16,float16,0,0.0495413343111674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,16,128,1,float16,float16,0,0.037530665596326195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,16,8,128,1,fp8,fp8,0,0.04632000128428141
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,16,128,1,float16,fp8,0,0.03738666574160258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,1,128,1,float16,float16,0,0.03515733281771342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,1,128,1,float16,fp8,0,0.03508266558249792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,1,128,1,fp8,fp8,0,0.030666666726271313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,2,128,1,float16,float16,0,0.035189333061377205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,2,128,1,fp8,fp8,0,0.03129599988460541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,2,128,1,float16,fp8,0,0.035743998984495796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,16,128,1,fp8,fp8,0,0.032229334115982056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,4,128,1,float16,fp8,0,0.035775999228159584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,4,128,1,float16,float16,0,0.036133334040641785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,4,128,1,fp8,fp8,0,0.0322026660044988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,8,128,1,float16,float16,0,0.03619199991226196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,8,128,1,fp8,fp8,0,0.03254933406909307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,16,8,128,1,float16,fp8,0,0.03656533360481262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,1,128,1,fp8,fp8,0,1.470677375793457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,1,128,1,float16,fp8,0,1.512122631072998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,1,128,1,float16,float16,0,1.522666613260905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,2,128,1,float16,float16,0,1.5357972780863445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,2,128,1,fp8,fp8,0,1.5506025950113933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,2,128,1,float16,fp8,0,1.5347946484883626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,4,128,1,float16,float16,0,1.6751786867777507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,4,128,1,float16,fp8,0,1.6796053250630696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,4,128,1,fp8,fp8,0,1.705301284790039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,16,128,1,float16,float16,0,1.0665067036946614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,16,128,1,float16,fp8,0,1.0261332988739014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,1,128,1,float16,float16,0,0.7701813379923502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,8,128,1,float16,float16,0,1.784000078837077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,8,128,1,fp8,fp8,0,1.7350452740987141
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,16,8,128,1,float16,fp8,0,1.7518399556477864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,1,128,1,float16,fp8,0,0.7720426718393961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,1,128,1,fp8,fp8,0,0.7277546723683676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,2,128,1,float16,float16,0,0.7823572953542074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,2,128,1,fp8,fp8,0,0.7866666316986084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,16,128,1,fp8,fp8,0,0.9369226296742758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,2,128,1,float16,fp8,0,0.779701312383016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,4,128,1,float16,float16,0,0.8232853412628174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,4,128,1,float16,fp8,0,0.8207253615061442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,4,128,1,fp8,fp8,0,0.86134934425354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,1,128,1,float16,float16,0,0.39842132727305096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,16,128,1,float16,float16,0,0.5379946629206339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,8,128,1,float16,float16,0,0.886303981145223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,8,128,1,fp8,fp8,0,0.8791306813557943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,16,8,128,1,float16,fp8,0,0.8786933422088623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,16,128,1,float16,fp8,0,0.5150560140609741
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,16,128,1,fp8,fp8,0,0.48073065280914307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,1,128,1,float16,fp8,0,0.3989706834157308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,1,128,1,fp8,fp8,0,0.37301333745320636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,2,128,1,float16,float16,0,0.40491199493408203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,2,128,1,float16,fp8,0,0.40316800276438397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,2,128,1,fp8,fp8,0,0.3985226551691691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,4,128,1,float16,fp8,0,0.41465600331624347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,4,128,1,fp8,fp8,0,0.44046934445699054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,4,128,1,float16,float16,0,0.4224853515625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,16,128,1,float16,float16,0,0.2789280017217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,8,128,1,float16,fp8,0,0.44629867871602374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,1,128,1,float16,float16,0,0.21166932582855225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,8,128,1,fp8,fp8,0,0.4487733443578084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,16,128,1,fp8,fp8,0,0.2481600046157837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,16,8,128,1,float16,float16,0,0.45185601711273193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,1,128,1,float16,fp8,0,0.21162132422129312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,16,128,1,float16,fp8,0,0.26946133375167847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,1,128,1,fp8,fp8,0,0.19861332575480142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,2,128,1,float16,float16,0,0.2149440050125122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,2,128,1,float16,fp8,0,0.2132426699002584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,2,128,1,fp8,fp8,0,0.2045066754023234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,4,128,1,float16,fp8,0,0.22209066152572632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,4,128,1,fp8,fp8,0,0.22817599773406982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,4,128,1,float16,float16,0,0.22361600399017334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,8,128,1,float16,float16,0,0.239029328028361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,8,128,1,fp8,fp8,0,0.23274133602778116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,16,128,1,float16,float16,0,0.15082133809725443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,16,8,128,1,float16,fp8,0,0.2341653307278951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,1,128,1,float16,fp8,0,0.11726933717727661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,16,128,1,float16,fp8,0,0.14647466937700906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,16,128,1,fp8,fp8,0,0.13215466340382895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,1,128,1,float16,float16,0,0.11822400490442912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,1,128,1,fp8,fp8,0,0.10875200231870015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,2,128,1,float16,float16,0,0.12053866187731425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,2,128,1,float16,fp8,0,0.11890133221944173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,2,128,1,fp8,fp8,0,0.11016000310579936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,4,128,1,float16,float16,0,0.12494933605194092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,4,128,1,float16,fp8,0,0.12463999787966411
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,8,128,1,float16,float16,0,0.13222933808962503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,4,128,1,fp8,fp8,0,0.12332800030708313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,8,128,1,float16,fp8,0,0.12929067015647888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,16,128,1,float16,float16,0,0.08737599849700928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,16,8,128,1,fp8,fp8,0,0.12065600355466206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,16,128,1,float16,fp8,0,0.08531199892361958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,16,128,1,fp8,fp8,0,0.07602666815121968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,1,128,1,float16,float16,0,0.06584000090758006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,1,128,1,fp8,fp8,0,0.06084266801675161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,2,128,1,float16,float16,0,0.06716800232728322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,1,128,1,float16,fp8,0,0.06628266473611195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,2,128,1,fp8,fp8,0,0.06292266647020976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,4,128,1,float16,float16,0,0.07098133365313213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,4,128,1,float16,fp8,0,0.07035199801127116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,2,128,1,float16,fp8,0,0.06739200154940288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,4,128,1,fp8,fp8,0,0.06717333197593689
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,8,128,1,float16,float16,0,0.0726506660381953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,16,128,1,float16,float16,0,0.04718933502833048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,8,128,1,fp8,fp8,0,0.06785066425800323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,16,128,1,float16,fp8,0,0.04611733555793762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,1,128,1,float16,float16,0,0.04002666721741358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,16,8,128,1,float16,fp8,0,0.0717493345340093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,1,128,1,float16,fp8,0,0.03994666785001755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,1,128,1,fp8,fp8,0,0.03629866739114126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,16,128,1,fp8,fp8,0,0.04477333525816599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,2,128,1,float16,float16,0,0.040048000713189445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,2,128,1,float16,fp8,0,0.04041066765785217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,2,128,1,fp8,fp8,0,0.03721066564321518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,4,128,1,float16,float16,0,0.041349334021409355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,4,128,1,float16,fp8,0,0.04167999823888143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,4,128,1,fp8,fp8,0,0.039120001097520195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,8,128,1,float16,fp8,0,0.04275733232498169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,8,128,1,fp8,fp8,0,0.04215466479460398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,16,128,1,float16,float16,0,0.03091199944416682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,16,8,128,1,float16,float16,0,0.04269866645336151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,16,128,1,float16,fp8,0,0.030858665704727173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,16,128,1,fp8,fp8,0,0.028229333460330963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,1,128,1,float16,float16,0,0.028250666956106823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,1,128,1,float16,fp8,0,0.028330666323502857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,1,128,1,fp8,fp8,0,0.026928000152111053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,2,128,1,float16,float16,0,0.028416000306606293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,2,128,1,fp8,fp8,0,0.026693334182103474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,2,128,1,float16,fp8,0,0.028373333315054577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,4,128,1,float16,float16,0,0.028751999139785767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,4,128,1,float16,fp8,0,0.029178666571776073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,4,128,1,fp8,fp8,0,0.028058665494124096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,8,128,1,float16,float16,0,0.029829333225886028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,8,128,1,float16,fp8,0,0.029135999580224354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,16,8,128,1,fp8,fp8,0,0.02847466617822647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,16,128,1,float16,fp8,0,0.02595199892918269
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,16,128,1,float16,float16,0,0.026762666801611584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,16,128,1,fp8,fp8,0,0.024469333390394848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,1,128,1,float16,float16,0,0.02566933383544286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,1,128,1,float16,fp8,0,0.02573866645495097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,1,128,1,fp8,fp8,0,0.02387733260790507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,2,128,1,float16,float16,0,0.025749333202838898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,2,128,1,float16,fp8,0,0.02584533393383026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,4,128,1,float16,fp8,0,0.026506667335828144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,4,128,1,float16,float16,0,0.025813333690166473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,2,128,1,fp8,fp8,0,0.023930666347344715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,4,128,1,fp8,fp8,0,0.024506665766239166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,8,128,1,float16,float16,0,0.025909334421157837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,8,128,1,fp8,fp8,0,0.02495466669400533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,16,8,128,1,float16,fp8,0,0.025994665920734406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,1,128,1,float16,float16,0,0.6650293270746866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,1,128,1,float16,fp8,0,0.6635093291600546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,1,128,1,fp8,fp8,0,0.6365866661071777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,2,128,1,float16,float16,0,0.6738239924112955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,2,128,1,float16,fp8,0,0.6703946590423584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,2,128,1,fp8,fp8,0,0.6925333340962728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,4,128,1,float16,float16,0,0.7244213422139486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,4,128,1,float16,fp8,0,0.7023999691009521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,4,128,1,fp8,fp8,0,0.7746880054473877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,16,128,1,float16,float16,0,0.4949013392130534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,8,128,1,float16,fp8,0,0.7645973364512125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,8,128,1,fp8,fp8,0,0.7892800172170004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,16,128,1,float16,fp8,0,0.4704586664835612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,16,8,128,1,float16,float16,0,0.7777439753214518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,1,128,1,float16,float16,0,0.3441813389460246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,1,128,1,float16,fp8,0,0.342410683631897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,1,128,1,fp8,fp8,0,0.3291626572608948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,2,128,1,float16,float16,0,0.3503040075302124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,2,128,1,float16,fp8,0,0.34669868151346844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,16,128,1,fp8,fp8,0,0.438042680422465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,2,128,1,fp8,fp8,0,0.3543039957682292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,4,128,1,float16,float16,0,0.36664533615112305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,4,128,1,float16,fp8,0,0.35971200466156006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,4,128,1,fp8,fp8,0,0.3956586519877116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,8,128,1,float16,float16,0,0.3989493449529012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,1,128,1,float16,float16,0,0.18237866957982382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,16,128,1,float16,fp8,0,0.24623467524846396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,16,128,1,float16,float16,0,0.2582559982935588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,8,128,1,float16,fp8,0,0.3880373239517212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,16,8,128,1,fp8,fp8,0,0.40249065558115643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,16,128,1,fp8,fp8,0,0.22723199923833212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,1,128,1,float16,fp8,0,0.1821546753247579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,1,128,1,fp8,fp8,0,0.1764906644821167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,2,128,1,float16,float16,0,0.1857653260231018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,2,128,1,float16,fp8,0,0.18448533614476523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,4,128,1,float16,float16,0,0.19552000363667807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,2,128,1,fp8,fp8,0,0.18497600158055624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,4,128,1,float16,fp8,0,0.19136534134546915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,4,128,1,fp8,fp8,0,0.2088800072669983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,8,128,1,float16,float16,0,0.21136534214019775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,1,128,1,float16,float16,0,0.10319466392199199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,8,128,1,float16,fp8,0,0.20547199249267578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,16,128,1,float16,float16,0,0.14064533511797586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,16,128,1,fp8,fp8,0,0.1225279966990153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,16,128,1,float16,fp8,0,0.13562666376431784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,16,8,128,1,fp8,fp8,0,0.21010132630666098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,1,128,1,float16,fp8,0,0.10317333539326985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,1,128,1,fp8,fp8,0,0.0997920036315918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,2,128,1,float16,fp8,0,0.10521599650382996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,2,128,1,fp8,fp8,0,0.0995360016822815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,2,128,1,float16,float16,0,0.10528533657391866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,4,128,1,float16,float16,0,0.10994133353233337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,4,128,1,float16,fp8,0,0.10873599847157796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,4,128,1,fp8,fp8,0,0.11114666859308879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,8,128,1,float16,float16,0,0.11687466502189636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,8,128,1,float16,fp8,0,0.11432000001271565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,16,128,1,float16,float16,0,0.08475733796755473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,16,8,128,1,fp8,fp8,0,0.11075733105341594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,1,128,1,float16,float16,0,0.060175999999046326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,1,128,1,float16,fp8,0,0.06015466650327047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,16,128,1,float16,fp8,0,0.08117866516113281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,1,128,1,fp8,fp8,0,0.056186666091283165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,2,128,1,float16,fp8,0,0.06105599800745646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,16,128,1,fp8,fp8,0,0.07108800113201141
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,2,128,1,fp8,fp8,0,0.05738666653633118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,4,128,1,float16,float16,0,0.06442666550477345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,2,128,1,float16,float16,0,0.061280002196629844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,4,128,1,fp8,fp8,0,0.06206933160622915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,8,128,1,float16,float16,0,0.06703466673692067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,4,128,1,float16,fp8,0,0.06408533453941345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,8,128,1,fp8,fp8,0,0.06317333380381267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,16,8,128,1,float16,fp8,0,0.06604266663392384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,16,128,1,float16,float16,0,0.044026667873064675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,1,128,1,float16,float16,0,0.03707200040419897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,16,128,1,fp8,fp8,0,0.04197866717974345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,16,128,1,float16,fp8,0,0.04316799839337667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,1,128,1,fp8,fp8,0,0.03322133421897888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,1,128,1,float16,fp8,0,0.03694933404525121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,2,128,1,float16,float16,0,0.0374293327331543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,2,128,1,float16,fp8,0,0.037808001041412354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,2,128,1,fp8,fp8,0,0.03410666684309641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,4,128,1,float16,float16,0,0.03904533386230469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,4,128,1,fp8,fp8,0,0.03664000084002813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,4,128,1,float16,fp8,0,0.038933334251244865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,16,128,1,float16,float16,0,0.02850666642189026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,8,128,1,float16,float16,0,0.040378667414188385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,16,128,1,float16,fp8,0,0.02784000088771184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,8,128,1,fp8,fp8,0,0.03817066550254822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,16,8,128,1,float16,fp8,0,0.039247999588648476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,16,128,1,fp8,fp8,0,0.026213333010673523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,1,128,1,float16,fp8,0,0.02550400048494339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,2,128,1,float16,float16,0,0.02607999990383784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,1,128,1,fp8,fp8,0,0.023013333479563396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,1,128,1,float16,float16,0,0.025994665920734406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,2,128,1,float16,fp8,0,0.026352000733216602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,4,128,1,float16,float16,0,0.027077332139015198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,4,128,1,float16,fp8,0,0.026517334083716076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,2,128,1,fp8,fp8,0,0.023599999646345775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,4,128,1,fp8,fp8,0,0.024773334463437397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,8,128,1,float16,float16,0,0.026949333647886913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,8,128,1,fp8,fp8,0,0.024469333390394848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,16,128,1,float16,fp8,0,0.023189333577950794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,16,128,1,float16,float16,0,0.023050665855407715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,1,128,1,float16,float16,0,0.02221333235502243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,16,8,128,1,float16,fp8,0,0.02754666656255722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,16,128,1,fp8,fp8,0,0.021253332495689392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,1,128,1,float16,fp8,0,0.02205866575241089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,1,128,1,fp8,fp8,0,0.020773333807786305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,2,128,1,float16,float16,0,0.022042666872342426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,2,128,1,float16,fp8,0,0.02214933435122172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,2,128,1,fp8,fp8,0,0.021189334491888683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,4,128,1,float16,float16,0,0.02232533444960912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,8,128,1,float16,float16,0,0.022800001005331676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,4,128,1,float16,fp8,0,0.022863999009132385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,8,128,1,float16,fp8,0,0.023178666830062866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,8,128,1,fp8,fp8,0,0.021520001192887623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,16,128,1,float16,float16,0,0.021589333812395733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,16,4,128,1,fp8,fp8,0,0.02111999938885371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,16,128,1,float16,fp8,0,0.02142400046189626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,16,128,1,fp8,fp8,0,0.019760000209013622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,1,128,1,float16,fp8,0,0.021082667013009388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,1,128,1,fp8,fp8,0,0.019493332753578823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,2,128,1,float16,float16,0,0.02144533395767212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,1,128,1,float16,float16,0,0.020869334538777668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,2,128,1,fp8,fp8,0,0.019946667055288952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,4,128,1,float16,float16,0,0.021045332153638203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,4,128,1,float16,fp8,0,0.02124800036350886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,2,128,1,float16,fp8,0,0.021221332252025604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,4,128,1,fp8,fp8,0,0.01989866668979327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,8,128,1,float16,float16,0,0.021370666722456615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,8,128,1,float16,fp8,0,0.021530665457248688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,16,8,128,1,fp8,fp8,0,0.020101333657900494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,1,128,1,float16,fp8,0,0.3179200092951457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,1,128,1,float16,float16,0,0.32344533999760944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,2,128,1,float16,fp8,0,0.3224053382873535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,2,128,1,float16,float16,0,0.32919466495513916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,1,128,1,fp8,fp8,0,0.3282559911410014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,2,128,1,fp8,fp8,0,0.35366400082906085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,4,128,1,float16,float16,0,0.34600532054901123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,4,128,1,float16,fp8,0,0.33752532800038654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,4,128,1,fp8,fp8,0,0.39500268300374347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,8,128,1,float16,float16,0,0.37267200152079266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,16,128,1,float16,float16,0,0.2631946603457133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,8,128,1,float16,fp8,0,0.36797865231831867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,16,128,1,float16,fp8,0,0.25296000639597577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,1,128,1,float16,float16,0,0.172325332959493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,16,8,128,1,fp8,fp8,0,0.4042880137761434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,1,128,1,float16,fp8,0,0.17160000403722128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,16,128,1,fp8,fp8,0,0.22570133209228516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,2,128,1,float16,float16,0,0.17806933323542276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,2,128,1,float16,fp8,0,0.174453337987264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,1,128,1,fp8,fp8,0,0.17538134256998697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,2,128,1,fp8,fp8,0,0.18581867218017578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,4,128,1,float16,float16,0,0.18743467330932617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,4,128,1,float16,fp8,0,0.18317866325378418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,4,128,1,fp8,fp8,0,0.20450133085250854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,8,128,1,float16,float16,0,0.2022613286972046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,16,128,1,float16,float16,0,0.1450933317343394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,8,128,1,float16,fp8,0,0.19647467136383057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,1,128,1,float16,float16,0,0.0997173289457957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,16,8,128,1,fp8,fp8,0,0.2084266742070516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,1,128,1,float16,fp8,0,0.09963732957839966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,1,128,1,fp8,fp8,0,0.09898666540781657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,2,128,1,float16,float16,0,0.10308266679445903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,16,128,1,fp8,fp8,0,0.12230400244394939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,16,128,1,float16,fp8,0,0.13939199844996134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,2,128,1,float16,fp8,0,0.10249599814414978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,2,128,1,fp8,fp8,0,0.10062399506568909
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,4,128,1,float16,float16,0,0.10884799559911092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,4,128,1,float16,fp8,0,0.10804800192515056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,4,128,1,fp8,fp8,0,0.11157332857449849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,8,128,1,float16,float16,0,0.11437333623568217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,8,128,1,float16,fp8,0,0.11190399527549744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,16,8,128,1,fp8,fp8,0,0.11237866679827373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,16,128,1,float16,float16,0,0.0798880010843277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,1,128,1,float16,float16,0,0.05754666527112325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,16,128,1,float16,fp8,0,0.07696533203125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,16,128,1,fp8,fp8,0,0.0703893353541692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,1,128,1,float16,fp8,0,0.058933332562446594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,2,128,1,float16,float16,0,0.05852266649405161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,1,128,1,fp8,fp8,0,0.05605333546797434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,4,128,1,float16,float16,0,0.06247999767462412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,4,128,1,float16,fp8,0,0.06161599854628245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,4,128,1,fp8,fp8,0,0.06252799928188324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,2,128,1,fp8,fp8,0,0.058229332168896995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,2,128,1,float16,fp8,0,0.05901333192984263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,8,128,1,float16,float16,0,0.06498133142789204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,16,128,1,float16,float16,0,0.04303466777006785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,8,128,1,fp8,fp8,0,0.06388799846172333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,16,128,1,float16,fp8,0,0.041722665230433144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,16,128,1,fp8,fp8,0,0.04204266766707102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,16,8,128,1,float16,fp8,0,0.06340266764163971
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,1,128,1,fp8,fp8,0,0.033786666889985405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,1,128,1,float16,fp8,0,0.03636800001064936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,2,128,1,float16,float16,0,0.037045332292715706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,2,128,1,float16,fp8,0,0.036506667733192444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,1,128,1,float16,float16,0,0.03619199991226196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,2,128,1,fp8,fp8,0,0.034703999757766724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,4,128,1,float16,fp8,0,0.03800000001986822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,4,128,1,fp8,fp8,0,0.037258667250474296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,8,128,1,float16,float16,0,0.03997866561015447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,4,128,1,float16,float16,0,0.03782933453718821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,8,128,1,float16,fp8,0,0.03815466662247976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,16,128,1,float16,float16,0,0.02731200059254964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,16,8,128,1,fp8,fp8,0,0.03843733419974645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,16,128,1,float16,fp8,0,0.027029333015282948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,1,128,1,float16,float16,0,0.025199999411900837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,16,128,1,fp8,fp8,0,0.025077333052953083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,1,128,1,fp8,fp8,0,0.023232000569502514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,2,128,1,float16,float16,0,0.025536000728607178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,1,128,1,float16,fp8,0,0.024906667570273083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,2,128,1,fp8,fp8,0,0.023226665953795116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,4,128,1,float16,float16,0,0.026575999955336254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,4,128,1,float16,fp8,0,0.025754667818546295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,4,128,1,fp8,fp8,0,0.02477866659561793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,8,128,1,float16,float16,0,0.02678400029738744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,2,128,1,float16,fp8,0,0.025616000096003216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,16,128,1,float16,float16,0,0.021514666577180225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,8,128,1,fp8,fp8,0,0.02480533222357432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,16,8,128,1,float16,fp8,0,0.026687999566396076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,16,128,1,float16,fp8,0,0.021295999487241108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,16,128,1,fp8,fp8,0,0.021402666966120403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,1,128,1,float16,float16,0,0.021312000850836437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,1,128,1,fp8,fp8,0,0.020618667205174763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,2,128,1,float16,float16,0,0.02103466788927714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,2,128,1,fp8,fp8,0,0.020773333807786305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,4,128,1,float16,float16,0,0.02111999938885371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,1,128,1,float16,fp8,0,0.021066665649414062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,4,128,1,float16,fp8,0,0.021664001047611237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,4,128,1,fp8,fp8,0,0.021514666577180225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,2,128,1,float16,fp8,0,0.02067733307679494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,8,128,1,float16,float16,0,0.021642667551835377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,8,128,1,float16,fp8,0,0.021210665504137676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,16,128,1,float16,fp8,0,0.019589333484570186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,16,8,128,1,fp8,fp8,0,0.02186666677395503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,16,128,1,fp8,fp8,0,0.01940800001223882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,1,128,1,float16,fp8,0,0.019648000597953796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,1,128,1,fp8,fp8,0,0.019546666493018467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,16,128,1,float16,float16,0,0.019199999670187633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,2,128,1,fp8,fp8,0,0.019941333681344986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,2,128,1,float16,float16,0,0.01960533360640208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,1,128,1,float16,float16,0,0.019541333119074505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,2,128,1,float16,fp8,0,0.020213333268960316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,8,128,1,float16,float16,0,0.01969066634774208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,4,128,1,float16,float16,0,0.01995733380317688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,4,128,1,float16,fp8,0,0.01995733380317688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,8,128,1,float16,fp8,0,0.02013333390156428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,16,128,1,float16,float16,0,0.019002666076024372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,4,128,1,fp8,fp8,0,0.019551999866962433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,16,128,1,float16,fp8,0,0.019146667172511418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,16,8,128,1,fp8,fp8,0,0.019861333072185516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,16,128,1,fp8,fp8,0,0.01912533367673556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,1,128,1,float16,float16,0,0.019173332800467808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,1,128,1,float16,fp8,0,0.01960533360640208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,1,128,1,fp8,fp8,0,0.019093333433071773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,2,128,1,float16,fp8,0,0.01966933285196622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,4,128,1,float16,float16,0,0.019173332800467808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,2,128,1,float16,float16,0,0.01886933296918869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,4,128,1,float16,fp8,0,0.019317333896954853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,4,128,1,fp8,fp8,0,0.019519999623298645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,8,128,1,float16,fp8,0,0.01953599974513054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,8,128,1,float16,float16,0,0.018725333114465077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,2,128,1,fp8,fp8,0,0.019018666197856266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,16,8,128,1,fp8,fp8,0,0.01926933353145917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,1,128,1,float16,fp8,0,0.20644799868265787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,1,128,1,float16,float16,0,0.20627200603485107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,1,128,1,fp8,fp8,0,0.22768000761667886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,2,128,1,float16,fp8,0,0.2100320061047872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,2,128,1,float16,float16,0,0.21207465728123984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,4,128,1,float16,float16,0,0.2230186661084493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,2,128,1,fp8,fp8,0,0.23642667134602866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,4,128,1,float16,fp8,0,0.219050665696462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,8,128,1,float16,float16,0,0.2363413373629252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,4,128,1,fp8,fp8,0,0.25920534133911133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,8,128,1,fp8,fp8,0,0.26291199525197345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,16,8,128,1,float16,fp8,0,0.23035200436909994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,16,128,1,float16,float16,0,0.1431893308957418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,16,128,1,float16,fp8,0,0.1381493310133616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,1,128,1,float16,float16,0,0.11353066563606262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,16,128,1,fp8,fp8,0,0.14779200156529745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,2,128,1,float16,float16,0,0.11749333143234253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,1,128,1,fp8,fp8,0,0.12390399972597758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,2,128,1,float16,fp8,0,0.11633599797884624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,1,128,1,float16,fp8,0,0.11381333072980244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,2,128,1,fp8,fp8,0,0.12678933143615723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,4,128,1,float16,float16,0,0.12477866808573405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,4,128,1,float16,fp8,0,0.1216159959634145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,4,128,1,fp8,fp8,0,0.13711466391881308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,8,128,1,float16,float16,0,0.13319999972979227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,8,128,1,float16,fp8,0,0.12874666849772134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,16,8,128,1,fp8,fp8,0,0.13779200116793314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,1,128,1,float16,float16,0,0.06473599870999654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,16,128,1,float16,fp8,0,0.07529599964618683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,1,128,1,float16,fp8,0,0.0645066648721695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,16,128,1,fp8,fp8,0,0.08277866741021474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,16,128,1,float16,float16,0,0.07881600161393483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,1,128,1,fp8,fp8,0,0.07005333403746287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,2,128,1,fp8,fp8,0,0.07217066486676534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,2,128,1,float16,fp8,0,0.0655626654624939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,4,128,1,float16,float16,0,0.06817600131034851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,4,128,1,float16,fp8,0,0.0682773341735204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,8,128,1,float16,float16,0,0.07252266506354015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,2,128,1,float16,float16,0,0.06611733138561249
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,4,128,1,fp8,fp8,0,0.07716266810894012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,8,128,1,float16,fp8,0,0.0701386680205663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,16,8,128,1,fp8,fp8,0,0.07755733529726665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,16,128,1,float16,float16,0,0.04378666480382284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,16,128,1,fp8,fp8,0,0.04896533489227295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,1,128,1,float16,float16,0,0.03888533264398575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,1,128,1,float16,fp8,0,0.038704000413417816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,16,128,1,float16,fp8,0,0.04242133100827535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,1,128,1,fp8,fp8,0,0.04188266893227895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,2,128,1,float16,float16,0,0.03946666667858759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,2,128,1,float16,fp8,0,0.039877332746982574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,2,128,1,fp8,fp8,0,0.04195199906826019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,4,128,1,fp8,fp8,0,0.04411733150482178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,8,128,1,float16,float16,0,0.04131199916203817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,4,128,1,float16,float16,0,0.04045866678158442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,8,128,1,float16,fp8,0,0.04092800120512644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,8,128,1,fp8,fp8,0,0.04537599782148997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,16,4,128,1,float16,fp8,0,0.04068800061941147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,16,128,1,float16,float16,0,0.028250666956106823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,16,128,1,float16,fp8,0,0.028064000109831493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,16,128,1,fp8,fp8,0,0.02978666623433431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,1,128,1,float16,fp8,0,0.026719999810059864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,1,128,1,fp8,fp8,0,0.027530667682488758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,2,128,1,float16,fp8,0,0.026778665681680042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,1,128,1,float16,float16,0,0.026357332865397137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,2,128,1,float16,float16,0,0.027050666511058807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,2,128,1,fp8,fp8,0,0.027850667635599773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,4,128,1,float16,float16,0,0.02808533360560735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,8,128,1,float16,float16,0,0.028031999866167705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,4,128,1,float16,fp8,0,0.02794666588306427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,8,128,1,fp8,fp8,0,0.028901333610216778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,8,128,1,float16,fp8,0,0.028234665592511494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,16,128,1,fp8,fp8,0,0.021402666966120403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,16,4,128,1,fp8,fp8,0,0.02886933336655299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,16,128,1,float16,float16,0,0.021136000752449036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,16,128,1,float16,fp8,0,0.02147199958562851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,1,128,1,float16,float16,0,0.020506666352351505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,2,128,1,float16,float16,0,0.020351999749739964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,2,128,1,fp8,fp8,0,0.020981334149837494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,4,128,1,float16,float16,0,0.02040533348917961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,1,128,1,float16,fp8,0,0.022463999688625336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,2,128,1,float16,fp8,0,0.020309332758188248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,4,128,1,float16,fp8,0,0.020746666938066483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,1,128,1,fp8,fp8,0,0.020495999604463577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,4,128,1,fp8,fp8,0,0.02146133283774058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,8,128,1,float16,float16,0,0.02082666630546252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,8,128,1,float16,fp8,0,0.020661332954963047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,1,128,1,float16,float16,0,0.01846933364868164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,16,128,1,float16,fp8,0,0.01911466692884763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,16,128,1,float16,float16,0,0.01838933303952217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,16,8,128,1,fp8,fp8,0,0.021354667842388153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,16,128,1,fp8,fp8,0,0.019541333119074505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,2,128,1,float16,float16,0,0.01859733338157336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,1,128,1,float16,fp8,0,0.018650667121013004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,1,128,1,fp8,fp8,0,0.019610666980346043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,2,128,1,float16,fp8,0,0.019199999670187633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,2,128,1,fp8,fp8,0,0.019823999454577763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,4,128,1,fp8,fp8,0,0.019717333217461903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,8,128,1,float16,float16,0,0.018650667121013004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,4,128,1,float16,fp8,0,0.018789333601792652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,4,128,1,float16,float16,0,0.018346666047970455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,8,128,1,float16,fp8,0,0.018858666221300762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,16,8,128,1,fp8,fp8,0,0.019978666057189304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,16,128,1,float16,fp8,0,0.01814933369557063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,16,128,1,float16,float16,0,0.01764800027012825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,16,128,1,fp8,fp8,0,0.018581333259741466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,1,128,1,fp8,fp8,0,0.019199999670187633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,2,128,1,float16,float16,0,0.01773333301146825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,2,128,1,float16,fp8,0,0.017850667238235474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,2,128,1,fp8,fp8,0,0.019013332823912304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,1,128,1,float16,fp8,0,0.018170667191346485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,1,128,1,float16,float16,0,0.017701332767804463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,4,128,1,float16,float16,0,0.018138666947682697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,4,128,1,float16,fp8,0,0.018522666146357853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,4,128,1,fp8,fp8,0,0.018826667219400406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,8,128,1,float16,float16,0,0.017802666872739792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,8,128,1,float16,fp8,0,0.01848000039656957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,16,8,128,1,fp8,fp8,0,0.026047999660174053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,16,128,1,float16,float16,0,0.01736533393462499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,16,128,1,float16,fp8,0,0.017573333034912746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,1,128,1,float16,float16,0,0.01743999992807706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,16,128,1,fp8,fp8,0,0.019002666076024372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,1,128,1,fp8,fp8,0,0.018613333503405254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,1,128,1,float16,fp8,0,0.017759999881188076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,2,128,1,fp8,fp8,0,0.01836799954374631
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,4,128,1,float16,float16,0,0.017658667018016178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,2,128,1,float16,float16,0,0.017312000195185345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,4,128,1,fp8,fp8,0,0.01863466699918111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,8,128,1,float16,float16,0,0.01757866640885671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,4,128,1,float16,fp8,0,0.018330667167901993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,2,128,1,float16,fp8,0,0.017994667092959087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,8,128,1,float16,fp8,0,0.017562666287024815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,16,8,128,1,fp8,fp8,0,0.019082666685183842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,1,128,1,float16,fp8,0,0.1471626659234365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,1,128,1,fp8,fp8,0,0.18499199549357095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,1,128,1,float16,float16,0,0.1467680037021637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,2,128,1,float16,float16,0,0.1521440049012502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,2,128,1,float16,fp8,0,0.1506666640440623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,2,128,1,fp8,fp8,0,0.18623999754587808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,4,128,1,float16,float16,0,0.15819199879964194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,4,128,1,float16,fp8,0,0.1569919983545939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,4,128,1,fp8,fp8,0,0.19719467560450235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,8,128,1,float16,fp8,0,0.16296533743540445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,16,128,1,float16,float16,0,0.0952959954738617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,16,128,1,float16,fp8,0,0.09404800335566203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,8,128,1,float16,float16,0,0.16420267025629678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,1,128,1,float16,float16,0,0.07898133496443431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,16,8,128,1,fp8,fp8,0,0.1980266571044922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,16,128,1,fp8,fp8,0,0.1144533356030782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,1,128,1,float16,fp8,0,0.07896533111731212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,1,128,1,fp8,fp8,0,0.10129066308339436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,2,128,1,float16,float16,0,0.08031466603279114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,2,128,1,fp8,fp8,0,0.10420266787211101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,2,128,1,float16,fp8,0,0.07980266710122426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,4,128,1,float16,fp8,0,0.0831413318713506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,4,128,1,float16,float16,0,0.08402132987976074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,4,128,1,fp8,fp8,0,0.10898133118947347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,8,128,1,float16,float16,0,0.0883146623770396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,8,128,1,float16,fp8,0,0.08763200044631958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,16,8,128,1,fp8,fp8,0,0.10950400431950887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,16,128,1,float16,fp8,0,0.0487360010544459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,1,128,1,float16,float16,0,0.04525866607824961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,16,128,1,float16,float16,0,0.05060799916585287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,16,128,1,fp8,fp8,0,0.06493866443634033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,1,128,1,float16,fp8,0,0.04553066690762838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,1,128,1,fp8,fp8,0,0.0568800022204717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,2,128,1,float16,float16,0,0.04557866851488749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,2,128,1,float16,fp8,0,0.046122665206591286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,2,128,1,fp8,fp8,0,0.058362667759259544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,8,128,1,float16,float16,0,0.04794133206208547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,4,128,1,fp8,fp8,0,0.06005866825580597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,8,128,1,float16,fp8,0,0.04799466828505198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,4,128,1,float16,fp8,0,0.047168001532554626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,4,128,1,float16,float16,0,0.04742933313051859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,16,8,128,1,fp8,fp8,0,0.061066667238871254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,16,128,1,float16,float16,0,0.031541332602500916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,16,128,1,fp8,fp8,0,0.036848001182079315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,1,128,1,float16,float16,0,0.030271999537944794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,2,128,1,float16,float16,0,0.03054933249950409
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,16,128,1,float16,fp8,0,0.031114667654037476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,1,128,1,fp8,fp8,0,0.03595733394225439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,2,128,1,float16,fp8,0,0.030917334059874218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,1,128,1,float16,fp8,0,0.030453334252039593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,2,128,1,fp8,fp8,0,0.03603733330965042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,4,128,1,float16,fp8,0,0.03138133386770884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,4,128,1,float16,float16,0,0.031125334401925404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,4,128,1,fp8,fp8,0,0.03740799923737844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,8,128,1,float16,fp8,0,0.03209600100914637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,8,128,1,float16,float16,0,0.030789333085219067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,16,128,1,float16,fp8,0,0.02260799954334895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,16,128,1,float16,float16,0,0.02219199885924657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,16,128,1,fp8,fp8,0,0.025637333591779072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,1,128,1,float16,fp8,0,0.022042666872342426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,1,128,1,float16,float16,0,0.02201066662867864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,16,8,128,1,fp8,fp8,0,0.036901332437992096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,1,128,1,fp8,fp8,0,0.024864000578721363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,2,128,1,fp8,fp8,0,0.02500266581773758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,2,128,1,float16,fp8,0,0.02204799900452296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,2,128,1,float16,float16,0,0.021903999149799347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,4,128,1,float16,float16,0,0.022090665996074677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,4,128,1,float16,fp8,0,0.022367998957633972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,4,128,1,fp8,fp8,0,0.025733334322770435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,8,128,1,float16,fp8,0,0.022789334257443745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,8,128,1,fp8,fp8,0,0.025706666211287182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,16,128,1,float16,float16,0,0.018426666657129925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,16,128,1,fp8,fp8,0,0.02029866725206375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,16,8,128,1,float16,float16,0,0.022437334060668945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,16,128,1,float16,fp8,0,0.01821333294113477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,1,128,1,float16,fp8,0,0.018112000077962875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,2,128,1,float16,float16,0,0.018170667191346485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,1,128,1,fp8,fp8,0,0.019498666127522785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,1,128,1,float16,float16,0,0.017765333255132038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,2,128,1,float16,fp8,0,0.018506667266289394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,4,128,1,float16,fp8,0,0.017968000223239262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,2,128,1,fp8,fp8,0,0.019999999552965164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,4,128,1,float16,float16,0,0.017727999637524288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,4,128,1,fp8,fp8,0,0.01985599969824155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,8,128,1,float16,float16,0,0.01823466643691063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,8,128,1,fp8,fp8,0,0.020181333025296528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,16,128,1,float16,float16,0,0.01695999999841054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,16,8,128,1,float16,fp8,0,0.018351999421914417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,1,128,1,float16,float16,0,0.01746133342385292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,16,128,1,fp8,fp8,0,0.019173332800467808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,1,128,1,float16,fp8,0,0.01762666677435239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,16,128,1,float16,fp8,0,0.01749333366751671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,1,128,1,fp8,fp8,0,0.018800000349680584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,2,128,1,fp8,fp8,0,0.019215999792019527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,2,128,1,float16,float16,0,0.01724799970785777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,4,128,1,float16,fp8,0,0.01782400036851565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,4,128,1,float16,float16,0,0.017456000049908955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,2,128,1,float16,fp8,0,0.01732800031701724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,8,128,1,float16,float16,0,0.01700266698996226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,4,128,1,fp8,fp8,0,0.018800000349680584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,8,128,1,float16,fp8,0,0.017680000513792038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,16,8,128,1,fp8,fp8,0,0.01878400022784869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,16,128,1,float16,float16,0,0.016623999923467636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,16,128,1,float16,fp8,0,0.017242666333913803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,1,128,1,float16,float16,0,0.016997333616018295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,1,128,1,fp8,fp8,0,0.0186666672428449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,2,128,1,float16,float16,0,0.017237332959969837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,16,128,1,fp8,fp8,0,0.0183999997874101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,1,128,1,float16,fp8,0,0.01699200024207433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,2,128,1,float16,fp8,0,0.016927999754746754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,2,128,1,fp8,fp8,0,0.019023999571800232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,4,128,1,float16,float16,0,0.017045332739750545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,4,128,1,float16,fp8,0,0.017184000462293625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,4,128,1,fp8,fp8,0,0.018672000616788864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,8,128,1,fp8,fp8,0,0.018170667191346485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,16,128,1,float16,float16,0,0.016623999923467636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,8,128,1,float16,float16,0,0.016783999900023144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,16,128,1,fp8,fp8,0,0.01807466646035512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,16,128,1,float16,fp8,0,0.017173333714405697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,16,8,128,1,float16,fp8,0,0.01724799970785777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,1,128,1,float16,float16,0,0.016949333250522614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,1,128,1,float16,fp8,0,0.01711999997496605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,1,128,1,fp8,fp8,0,0.018687999496857326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,2,128,1,float16,fp8,0,0.01703466723362605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,2,128,1,fp8,fp8,0,0.01877333347996076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,4,128,1,float16,fp8,0,0.017375999440749485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,2,128,1,float16,float16,0,0.016704000532627106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,4,128,1,float16,float16,0,0.016447999825080235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,4,128,1,fp8,fp8,0,0.018506667266289394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,8,128,1,float16,fp8,0,0.017317333569129307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,8,128,1,float16,float16,0,0.016682667036851246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,16,8,128,1,fp8,fp8,0,0.018853332847356796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,1,128,1,float16,float16,0,0.12246400117874146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,1,128,1,fp8,fp8,0,0.16160000363985697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,2,128,1,float16,fp8,0,0.12359467148780823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,2,128,1,float16,float16,0,0.12357866764068604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,1,128,1,float16,fp8,0,0.12282133102416992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,2,128,1,fp8,fp8,0,0.16410133242607117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,4,128,1,float16,float16,0,0.12763733665148416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,4,128,1,float16,fp8,0,0.12641599774360657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,16,128,1,float16,float16,0,0.0695360004901886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,8,128,1,float16,float16,0,0.13140267133712769
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,8,128,1,float16,fp8,0,0.12957333525021872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,16,128,1,float16,fp8,0,0.06780800223350525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,4,128,1,fp8,fp8,0,0.1687999963760376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,16,8,128,1,fp8,fp8,0,0.17062934239705405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,16,128,1,fp8,fp8,0,0.09643200039863586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,1,128,1,float16,float16,0,0.06715199848016103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,1,128,1,float16,fp8,0,0.06779733300209045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,1,128,1,fp8,fp8,0,0.08847999572753906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,2,128,1,float16,float16,0,0.06783999999364217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,2,128,1,float16,fp8,0,0.06778133412202199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,4,128,1,float16,float16,0,0.0685280015071233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,2,128,1,fp8,fp8,0,0.0904853343963623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,4,128,1,float16,fp8,0,0.06932266553243001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,4,128,1,fp8,fp8,0,0.09233066439628601
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,8,128,1,float16,float16,0,0.07049066821734111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,8,128,1,float16,fp8,0,0.06919999917348225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,16,128,1,float16,float16,0,0.040336000422636666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,16,8,128,1,fp8,fp8,0,0.09276800354321797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,1,128,1,float16,float16,0,0.04107200105985006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,16,128,1,float16,fp8,0,0.03969600051641464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,2,128,1,float16,float16,0,0.04154666761557261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,1,128,1,fp8,fp8,0,0.05162666738033295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,16,128,1,fp8,fp8,0,0.05286933481693268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,1,128,1,float16,fp8,0,0.04098666707674662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,2,128,1,fp8,fp8,0,0.05202666421731313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,4,128,1,float16,fp8,0,0.04204266766707102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,2,128,1,float16,fp8,0,0.041402667760849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,4,128,1,fp8,fp8,0,0.05302399893601736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,8,128,1,float16,float16,0,0.042277331153551735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,4,128,1,float16,float16,0,0.04132800052563349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,8,128,1,float16,fp8,0,0.04158399999141693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,16,8,128,1,fp8,fp8,0,0.05312533179918925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,16,128,1,float16,float16,0,0.027647999425729115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,16,128,1,fp8,fp8,0,0.03367999941110611
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,16,128,1,float16,fp8,0,0.02697066714366277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,1,128,1,float16,float16,0,0.028101332485675812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,1,128,1,fp8,fp8,0,0.03278400003910065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,2,128,1,float16,float16,0,0.027850667635599773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,1,128,1,float16,fp8,0,0.0273333340883255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,2,128,1,float16,fp8,0,0.028330666323502857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,2,128,1,fp8,fp8,0,0.033626665671666466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,4,128,1,float16,fp8,0,0.028602667152881622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,4,128,1,float16,float16,0,0.02752000093460083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,8,128,1,float16,float16,0,0.028250666956106823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,4,128,1,fp8,fp8,0,0.033759998778502144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,8,128,1,fp8,fp8,0,0.03391999999682108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,16,128,1,float16,float16,0,0.020714666694402695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,16,8,128,1,float16,fp8,0,0.02811199923356374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,16,128,1,float16,fp8,0,0.020768000433842342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,1,128,1,float16,float16,0,0.020560000091791153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,1,128,1,float16,fp8,0,0.020975999534130096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,16,128,1,fp8,fp8,0,0.024319998919963837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,1,128,1,fp8,fp8,0,0.023503998915354412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,2,128,1,float16,float16,0,0.02120000123977661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,2,128,1,float16,fp8,0,0.0210506667693456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,4,128,1,float16,float16,0,0.020975999534130096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,2,128,1,fp8,fp8,0,0.023589332898457844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,4,128,1,float16,fp8,0,0.021040000021457672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,8,128,1,float16,float16,0,0.02067199970285098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,4,128,1,fp8,fp8,0,0.02382933348417282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,8,128,1,float16,fp8,0,0.021509334444999695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,16,128,1,float16,float16,0,0.018186666071414948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,16,128,1,float16,fp8,0,0.018245333184798557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,16,128,1,fp8,fp8,0,0.019920000185569126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,1,128,1,float16,float16,0,0.01714666684468587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,16,8,128,1,fp8,fp8,0,0.02421333392461141
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,1,128,1,float16,fp8,0,0.01754133279124896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,1,128,1,fp8,fp8,0,0.018895999838908512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,2,128,1,float16,float16,0,0.017797333498795826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,4,128,1,float16,float16,0,0.01730666682124138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,2,128,1,fp8,fp8,0,0.01913600042462349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,2,128,1,float16,fp8,0,0.017866666118303936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,4,128,1,fp8,fp8,0,0.01942933350801468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,8,128,1,float16,float16,0,0.017173333714405697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,4,128,1,float16,fp8,0,0.017802666872739792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,8,128,1,fp8,fp8,0,0.019813333948453266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,16,8,128,1,float16,fp8,0,0.018272000054518383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,16,128,1,float16,float16,0,0.01728533332546552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,16,128,1,float16,fp8,0,0.017829333742459614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,1,128,1,float16,float16,0,0.016901332885026932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,1,128,1,fp8,fp8,0,0.01870399961868922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,16,128,1,fp8,fp8,0,0.018650667121013004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,2,128,1,float16,float16,0,0.016837333639462788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,2,128,1,float16,fp8,0,0.01739199956258138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,2,128,1,fp8,fp8,0,0.01897066707412402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,1,128,1,float16,fp8,0,0.017237332959969837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,4,128,1,float16,fp8,0,0.01754133279124896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,4,128,1,float16,float16,0,0.016869333883126576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,8,128,1,float16,fp8,0,0.017125333348910015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,8,128,1,fp8,fp8,0,0.019152000546455383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,4,128,1,fp8,fp8,0,0.01855466639002164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,16,8,128,1,float16,float16,0,0.016864000509182613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,16,128,1,float16,fp8,0,0.01721599946419398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,16,128,1,float16,float16,0,0.016901332885026932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,16,128,1,fp8,fp8,0,0.01860800012946129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,1,128,1,float16,float16,0,0.016480000068744022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,1,128,1,float16,fp8,0,0.017610666652520496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,2,128,1,float16,float16,0,0.016842667013406754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,2,128,1,float16,fp8,0,0.017423999806245167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,1,128,1,fp8,fp8,0,0.018592000007629395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,4,128,1,float16,fp8,0,0.01720000058412552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,4,128,1,float16,float16,0,0.016976000120242436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,2,128,1,fp8,fp8,0,0.01859733338157336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,4,128,1,fp8,fp8,0,0.018789333601792652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,8,128,1,float16,fp8,0,0.017152000218629837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,8,128,1,float16,float16,0,0.016949333250522614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,16,8,128,1,fp8,fp8,0,0.0183999997874101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,16,128,1,float16,float16,0,0.016117333124081295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,16,128,1,float16,fp8,0,0.01693333312869072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,1,128,1,float16,float16,0,0.016751999656359356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,1,128,1,float16,fp8,0,0.016650666793187458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,1,128,1,fp8,fp8,0,0.018351999421914417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,16,128,1,fp8,fp8,0,0.0182239996890227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,2,128,1,float16,float16,0,0.016282666474580765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,2,128,1,float16,fp8,0,0.01706133286158244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,4,128,1,float16,float16,0,0.016447999825080235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,4,128,1,float16,fp8,0,0.01695999999841054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,2,128,1,fp8,fp8,0,0.018944000204404194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,4,128,1,fp8,fp8,0,0.018650667121013004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,8,128,1,fp8,fp8,0,0.01859733338157336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,8,128,1,float16,float16,0,0.016447999825080235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,16,8,128,1,float16,fp8,0,0.016976000120242436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,12,1,128,1,float16,fp8,0,12.601861317952475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,12,1,128,1,float16,float16,0,12.661242167154947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,12,1,128,1,fp8,fp8,0,8.06932258605957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,12,2,128,1,fp8,fp8,0,8.293989181518555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,12,2,128,1,float16,fp8,0,12.873189290364584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,12,2,128,1,float16,float16,0,12.785728454589844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,12,4,128,1,float16,float16,0,12.605562845865885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,1,128,1,float16,float16,0,6.25703493754069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,1,128,1,float16,fp8,0,6.053818384806315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,12,128,1,float16,float16,0,6.069429397583008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,12,4,128,1,fp8,fp8,0,8.254186630249023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,12,4,128,1,float16,fp8,0,12.678512573242188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,1,128,1,fp8,fp8,0,4.063376108805339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,12,128,1,fp8,fp8,0,4.0854291915893555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,12,128,1,float16,fp8,0,6.378000259399414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,2,128,1,float16,float16,0,6.104149500528972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,2,128,1,float16,fp8,0,6.166271845499675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,2,128,1,fp8,fp8,0,4.0370025634765625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,4,128,1,float16,fp8,0,6.1696319580078125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,4,128,1,float16,float16,0,6.358021418253581
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,1,128,1,float16,float16,0,3.0823520024617515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,12,128,1,float16,float16,0,3.0786612828572593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,12,4,128,1,fp8,fp8,0,4.0743147532145185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,1,128,1,fp8,fp8,0,2.098384062449137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,1,128,1,float16,fp8,0,3.040837287902832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,12,128,1,float16,fp8,0,3.0856053034464517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,12,128,1,fp8,fp8,0,2.111018657684326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,2,128,1,float16,float16,0,3.0732428232828775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,2,128,1,float16,fp8,0,3.04695987701416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,2,128,1,fp8,fp8,0,2.1058079401652017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,4,128,1,float16,float16,0,3.0909334818522134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,1,128,1,float16,float16,0,1.6676534016927083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,4,128,1,float16,fp8,0,3.054485321044922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,12,4,128,1,fp8,fp8,0,2.1128427187601724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,12,128,1,float16,float16,0,1.6173760096232097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,12,128,1,float16,fp8,0,1.6207253138224285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,12,128,1,fp8,fp8,0,1.0671679973602295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,1,128,1,float16,fp8,0,1.6356746355692546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,1,128,1,fp8,fp8,0,1.0619573593139648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,2,128,1,float16,float16,0,1.6451892852783203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,2,128,1,fp8,fp8,0,1.0655146439870198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,2,128,1,float16,fp8,0,1.6318613688151042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,4,128,1,float16,float16,0,1.6505066553751628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,4,128,1,float16,fp8,0,1.6372586886088054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,12,4,128,1,fp8,fp8,0,1.0705066521962483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,12,1,128,1,float16,float16,0,7.512016296386719
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,12,1,128,1,float16,fp8,0,7.165168126424153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,12,1,128,1,fp8,fp8,0,4.687594731648763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,12,2,128,1,float16,float16,0,7.128773371378581
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,12,2,128,1,float16,fp8,0,7.358965555826823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,12,2,128,1,fp8,fp8,0,4.699295997619629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,12,4,128,1,float16,float16,0,7.386575698852539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,1,128,1,float16,float16,0,3.4795519510904946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,12,128,1,float16,float16,0,3.483029365539551
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,1,128,1,float16,fp8,0,3.5327466328938804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,12,4,128,1,fp8,fp8,0,4.783189455668132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,12,4,128,1,float16,fp8,0,7.214890797932942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,12,128,1,fp8,fp8,0,2.451770623524984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,12,128,1,float16,fp8,0,3.4375413258870444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,1,128,1,fp8,fp8,0,2.411498705546061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,2,128,1,float16,float16,0,3.531365394592285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,2,128,1,fp8,fp8,0,2.426901340484619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,2,128,1,float16,fp8,0,3.474757194519043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,4,128,1,float16,float16,0,3.6012531916300454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,4,128,1,float16,fp8,0,3.5609172185262046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,12,4,128,1,fp8,fp8,0,2.441354592641195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,1,128,1,float16,float16,0,1.8268213272094727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,1,128,1,float16,fp8,0,1.8357067108154297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,12,128,1,float16,float16,0,1.8136213620503743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,1,128,1,fp8,fp8,0,1.2737973531087239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,12,128,1,float16,fp8,0,1.7967146237691243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,12,128,1,fp8,fp8,0,1.293573300043742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,2,128,1,float16,float16,0,1.8217333157857258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,2,128,1,float16,fp8,0,1.8158453305562336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,2,128,1,fp8,fp8,0,1.2784852981567383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,4,128,1,float16,float16,0,1.8309280077616374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,4,128,1,float16,fp8,0,1.818501313527425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,12,4,128,1,fp8,fp8,0,1.2865493297576904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,12,128,1,float16,float16,0,0.990997314453125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,12,128,1,float16,fp8,0,0.9811946551005045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,12,128,1,fp8,fp8,0,0.6571626663208008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,1,128,1,float16,float16,0,1.0109333197275798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,1,128,1,fp8,fp8,0,0.6487199862798055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,1,128,1,float16,fp8,0,0.9921120007832845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,2,128,1,float16,float16,0,1.0029760201772053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,2,128,1,float16,fp8,0,0.9967679977416992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,4,128,1,float16,float16,0,1.00654935836792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,2,128,1,fp8,fp8,0,0.6512853304545084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,4,128,1,fp8,fp8,0,0.6551413138707479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,12,4,128,1,float16,fp8,0,1.0088266531626384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,12,1,128,1,float16,float16,0,5.112736066182454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,12,1,128,1,float16,fp8,0,5.043525377909343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,12,1,128,1,fp8,fp8,0,3.399541219075521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,12,2,128,1,float16,float16,0,4.925605456034343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,12,2,128,1,float16,fp8,0,4.817034721374512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,12,2,128,1,fp8,fp8,0,3.407658576965332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,12,4,128,1,float16,float16,0,5.107632001241048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,1,128,1,float16,float16,0,2.4988320668538413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,12,128,1,float16,float16,0,2.502554734547933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,12,4,128,1,fp8,fp8,0,3.434783935546875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,1,128,1,float16,fp8,0,2.5181172688802085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,12,4,128,1,float16,fp8,0,4.878085454305013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,12,128,1,float16,fp8,0,2.5011733373006186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,12,128,1,fp8,fp8,0,1.794432004292806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,1,128,1,fp8,fp8,0,1.7560213406880696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,2,128,1,float16,float16,0,2.497135957082113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,2,128,1,float16,fp8,0,2.4713279406229653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,2,128,1,fp8,fp8,0,1.7624692916870117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,4,128,1,float16,float16,0,2.5096054077148438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,4,128,1,float16,fp8,0,2.5143680572509766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,1,128,1,float16,float16,0,1.3366026878356934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,12,128,1,float16,float16,0,1.321168025334676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,12,4,128,1,fp8,fp8,0,1.7773547172546387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,1,128,1,float16,fp8,0,1.3041919867197673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,12,128,1,float16,fp8,0,1.3072906335194905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,1,128,1,fp8,fp8,0,0.9368000030517578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,12,128,1,fp8,fp8,0,0.9568906625111898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,2,128,1,float16,float16,0,1.3404746055603027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,2,128,1,float16,fp8,0,1.3119146823883057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,2,128,1,fp8,fp8,0,0.9401493072509766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,4,128,1,float16,float16,0,1.3253813584645588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,4,128,1,float16,fp8,0,1.314522663752238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,12,4,128,1,fp8,fp8,0,0.945792039235433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,12,128,1,float16,float16,0,0.7302772998809814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,12,128,1,float16,fp8,0,0.7348586718241373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,1,128,1,float16,float16,0,0.7371306419372559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,12,128,1,fp8,fp8,0,0.4873280127843221
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,1,128,1,fp8,fp8,0,0.4794400135676066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,1,128,1,float16,fp8,0,0.7299306392669678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,2,128,1,float16,float16,0,0.7413919766743978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,2,128,1,float16,fp8,0,0.7338879903157552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,2,128,1,fp8,fp8,0,0.48106666405995685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,4,128,1,float16,fp8,0,0.7353920141855875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,4,128,1,float16,float16,0,0.740880012512207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,12,4,128,1,fp8,fp8,0,0.4861226479212443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,12,1,128,1,float16,float16,0,6.77072016398112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,12,1,128,1,fp8,fp8,0,4.520890553792317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,12,1,128,1,float16,fp8,0,6.792218526204427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,12,2,128,1,fp8,fp8,0,4.534757296244304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,12,2,128,1,float16,fp8,0,6.599434534708659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,12,2,128,1,float16,float16,0,6.815845489501953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,12,4,128,1,float16,float16,0,6.733445485432942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,1,128,1,float16,float16,0,3.3678986231486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,1,128,1,float16,fp8,0,3.239311854044596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,12,128,1,float16,float16,0,3.304133415222168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,12,4,128,1,fp8,fp8,0,4.616346677144368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,12,4,128,1,float16,fp8,0,6.782512029012044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,12,128,1,float16,fp8,0,3.3545920054117837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,1,128,1,fp8,fp8,0,2.314906597137451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,12,128,1,fp8,fp8,0,2.3729653358459473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,2,128,1,float16,float16,0,3.254762649536133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,2,128,1,float16,fp8,0,3.234090805053711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,2,128,1,fp8,fp8,0,2.321983973185221
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,12,128,1,float16,float16,0,1.6957119305928547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,4,128,1,float16,float16,0,3.2742134730021157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,4,128,1,fp8,fp8,0,2.3420373598734536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,1,128,1,float16,float16,0,1.672111988067627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,12,4,128,1,float16,fp8,0,3.2279841105143228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,12,128,1,float16,fp8,0,1.6757814089457195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,12,128,1,fp8,fp8,0,1.2397440274556477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,1,128,1,float16,fp8,0,1.6578772862752278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,2,128,1,float16,float16,0,1.704576015472412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,2,128,1,float16,fp8,0,1.6572532653808594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,1,128,1,fp8,fp8,0,1.1994880040486653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,2,128,1,fp8,fp8,0,1.2060693105061848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,4,128,1,float16,float16,0,1.6861119270324707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,4,128,1,float16,fp8,0,1.6738452911376953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,1,128,1,float16,float16,0,0.8991519610087076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,12,4,128,1,fp8,fp8,0,1.218021313349406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,12,128,1,float16,float16,0,0.9138080279032389
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,12,128,1,float16,fp8,0,0.8904159863789877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,1,128,1,float16,fp8,0,0.8895359834035238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,12,128,1,fp8,fp8,0,0.6312479972839355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,1,128,1,fp8,fp8,0,0.6097546815872192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,2,128,1,float16,float16,0,0.9027093251546224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,2,128,1,fp8,fp8,0,0.614026665687561
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,2,128,1,float16,fp8,0,0.8916160265604655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,4,128,1,float16,float16,0,0.905290683110555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,4,128,1,float16,fp8,0,0.896613359451294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,12,128,1,float16,float16,0,0.4522453149159749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,12,4,128,1,fp8,fp8,0,0.6213226715723673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,1,128,1,float16,float16,0,0.44602131843566895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,1,128,1,float16,fp8,0,0.4407573143641154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,12,128,1,float16,fp8,0,0.44364798069000244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,1,128,1,fp8,fp8,0,0.33554665247599286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,12,128,1,fp8,fp8,0,0.3463946580886841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,2,128,1,float16,float16,0,0.44699732462565106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,2,128,1,fp8,fp8,0,0.3371200164159139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,4,128,1,float16,float16,0,0.44919999440511066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,2,128,1,float16,fp8,0,0.4409866730372111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,4,128,1,fp8,fp8,0,0.3405653238296509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,12,4,128,1,float16,fp8,0,0.4421546856562297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,12,1,128,1,float16,fp8,0,3.889333407084147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,12,1,128,1,fp8,fp8,0,2.7813491821289062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,12,1,128,1,float16,float16,0,3.9788640340169272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,12,2,128,1,float16,float16,0,3.828677177429199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,12,2,128,1,fp8,fp8,0,2.7968320846557617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,12,2,128,1,float16,fp8,0,3.784559885660807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,12,4,128,1,float16,float16,0,3.9276533126831055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,1,128,1,float16,float16,0,1.9479093551635742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,1,128,1,float16,fp8,0,1.919808069864909
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,12,4,128,1,float16,fp8,0,3.8331146240234375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,12,128,1,fp8,fp8,0,1.4869813919067383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,12,128,1,float16,fp8,0,1.963930606842041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,12,4,128,1,fp8,fp8,0,2.8264106114705405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,1,128,1,fp8,fp8,0,1.4284799893697102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,12,128,1,float16,float16,0,2.0294559796651206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,2,128,1,float16,float16,0,1.9518879254659016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,2,128,1,float16,fp8,0,1.9352906545003254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,2,128,1,fp8,fp8,0,1.438645362854004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,4,128,1,float16,float16,0,1.965712070465088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,4,128,1,float16,fp8,0,1.9447360038757324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,12,4,128,1,fp8,fp8,0,1.4518933296203613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,12,128,1,float16,float16,0,1.0518080393473308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,1,128,1,float16,float16,0,1.018506685892741
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,1,128,1,float16,fp8,0,1.0051999886830647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,12,128,1,float16,fp8,0,1.021018664042155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,1,128,1,fp8,fp8,0,0.7517759799957275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,2,128,1,float16,float16,0,1.0214719772338867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,12,128,1,fp8,fp8,0,0.7843146324157715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,2,128,1,float16,fp8,0,1.020357370376587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,2,128,1,fp8,fp8,0,0.7571626504262289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,4,128,1,float16,float16,0,1.0274720191955566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,4,128,1,fp8,fp8,0,0.7656160195668539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,12,4,128,1,float16,fp8,0,1.0146613121032715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,12,128,1,float16,float16,0,0.5626879930496216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,12,128,1,float16,fp8,0,0.5590986808141073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,1,128,1,float16,float16,0,0.5613439877827963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,12,128,1,fp8,fp8,0,0.40591466426849365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,1,128,1,float16,fp8,0,0.5524106820424398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,1,128,1,fp8,fp8,0,0.3859359820683797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,2,128,1,float16,fp8,0,0.5547680060068766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,2,128,1,fp8,fp8,0,0.38782934347788495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,2,128,1,float16,float16,0,0.5628639856974283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,4,128,1,float16,float16,0,0.5650560061136881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,4,128,1,float16,fp8,0,0.5607093175252279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,12,4,128,1,fp8,fp8,0,0.39414934317270917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,12,128,1,float16,float16,0,0.2868959903717041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,12,128,1,float16,fp8,0,0.28443199396133423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,1,128,1,float16,float16,0,0.2826720078786214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,1,128,1,float16,fp8,0,0.2781066695849101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,12,128,1,fp8,fp8,0,0.22748800118764242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,2,128,1,float16,float16,0,0.28384000062942505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,2,128,1,float16,fp8,0,0.27905066808064777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,2,128,1,fp8,fp8,0,0.22116265694300333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,4,128,1,float16,fp8,0,0.28005866209665936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,4,128,1,fp8,fp8,0,0.22331732511520386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,4,128,1,float16,float16,0,0.2840106685956319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,12,1,128,1,fp8,fp8,0,0.21991999944051108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,12,1,128,1,float16,float16,0,3.7908960978190103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,12,1,128,1,float16,fp8,0,3.7458985646565757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,12,1,128,1,fp8,fp8,0,2.855226516723633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,12,2,128,1,float16,float16,0,3.8124958674112954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,12,2,128,1,float16,fp8,0,3.816495895385742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,12,2,128,1,fp8,fp8,0,2.872197469075521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,12,4,128,1,float16,float16,0,3.9348586400349936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,1,128,1,float16,float16,0,1.9046239852905273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,1,128,1,float16,fp8,0,1.898047924041748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,12,128,1,float16,float16,0,1.9840213457743328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,12,128,1,float16,fp8,0,1.9496639569600422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,12,4,128,1,fp8,fp8,0,2.91103458404541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,1,128,1,fp8,fp8,0,1.4537280400594075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,12,128,1,fp8,fp8,0,1.5393226941426594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,12,4,128,1,float16,fp8,0,3.779717445373535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,2,128,1,float16,float16,0,1.94267733891805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,2,128,1,float16,fp8,0,1.8798240025838215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,2,128,1,fp8,fp8,0,1.4642400741577148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,4,128,1,float16,float16,0,1.9361012776692708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,12,128,1,float16,float16,0,1.016213337580363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,4,128,1,float16,fp8,0,1.8980266253153484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,12,4,128,1,fp8,fp8,0,1.483290672302246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,1,128,1,float16,float16,0,0.9820480346679688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,1,128,1,float16,fp8,0,0.9625386397043864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,12,128,1,float16,fp8,0,1.0185333093007405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,2,128,1,float16,fp8,0,0.9676106770833334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,1,128,1,fp8,fp8,0,0.7540106773376465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,2,128,1,float16,float16,0,0.9840266704559326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,12,128,1,fp8,fp8,0,0.7970133622487386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,2,128,1,fp8,fp8,0,0.7578667004903158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,1,128,1,float16,float16,0,0.525493343671163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,4,128,1,float16,float16,0,0.994208017985026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,4,128,1,fp8,fp8,0,0.7667146523793539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,12,128,1,float16,float16,0,0.5396213531494141
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,12,128,1,float16,fp8,0,0.5315413475036621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,12,4,128,1,float16,fp8,0,0.9852000077565511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,12,128,1,fp8,fp8,0,0.4145546754201253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,1,128,1,float16,fp8,0,0.51801065603892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,1,128,1,fp8,fp8,0,0.38414398829142254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,2,128,1,float16,float16,0,0.5277173519134521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,2,128,1,float16,fp8,0,0.5201866626739502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,4,128,1,float16,float16,0,0.5336266756057739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,2,128,1,fp8,fp8,0,0.3858559926350911
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,4,128,1,float16,fp8,0,0.5244906743367513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,12,128,1,float16,float16,0,0.27747732400894165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,12,4,128,1,fp8,fp8,0,0.39483733971913654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,12,128,1,float16,fp8,0,0.27395200729370117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,12,128,1,fp8,fp8,0,0.2267306645711263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,1,128,1,float16,float16,0,0.26623467604319256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,1,128,1,fp8,fp8,0,0.21257599194844565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,1,128,1,float16,fp8,0,0.2606719930966695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,2,128,1,float16,float16,0,0.26633065938949585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,2,128,1,float16,fp8,0,0.26243199904759723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,2,128,1,fp8,fp8,0,0.2134666641553243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,4,128,1,float16,float16,0,0.2699093421300252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,4,128,1,fp8,fp8,0,0.21661333243052164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,12,4,128,1,float16,fp8,0,0.26553599039713544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,12,128,1,fp8,fp8,0,0.1341493328412374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,12,128,1,float16,fp8,0,0.15967466433842978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,1,128,1,float16,float16,0,0.15566933155059814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,12,128,1,float16,float16,0,0.16120533148447672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,1,128,1,float16,fp8,0,0.15408000349998474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,1,128,1,fp8,fp8,0,0.12424533565839131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,2,128,1,float16,fp8,0,0.1546986699104309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,4,128,1,float16,float16,0,0.15829333662986755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,2,128,1,fp8,fp8,0,0.12586667140324911
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,4,128,1,fp8,fp8,0,0.12989333271980286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,4,128,1,float16,fp8,0,0.15574399630228677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,12,2,128,1,float16,float16,0,0.15643200278282166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,12,1,128,1,float16,float16,0,2.3470346132914224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,12,1,128,1,fp8,fp8,0,1.836901346842448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,12,1,128,1,float16,fp8,0,2.291173299153646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,12,2,128,1,float16,fp8,0,2.3049333890279136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,12,2,128,1,float16,float16,0,2.3635679880777993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,12,2,128,1,fp8,fp8,0,1.852901299794515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,12,4,128,1,float16,float16,0,2.434586683909098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,1,128,1,float16,float16,0,1.185365358988444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,1,128,1,float16,fp8,0,1.1715892950693767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,12,128,1,float16,float16,0,1.2538666725158691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,12,4,128,1,float16,fp8,0,2.3359840710957847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,12,4,128,1,fp8,fp8,0,1.8802080154418945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,12,128,1,float16,fp8,0,1.2279040018717449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,12,128,1,fp8,fp8,0,1.009173313776652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,1,128,1,fp8,fp8,0,0.9382507006327311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,2,128,1,float16,float16,0,1.204207976659139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,2,128,1,float16,fp8,0,1.1678240299224854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,2,128,1,fp8,fp8,0,0.9444639682769775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,4,128,1,float16,float16,0,1.2095253467559814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,4,128,1,float16,fp8,0,1.1818933486938477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,12,4,128,1,fp8,fp8,0,0.9614346822102865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,12,128,1,float16,float16,0,0.650816003481547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,1,128,1,float16,float16,0,0.6180533170700073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,1,128,1,fp8,fp8,0,0.4909173250198364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,1,128,1,float16,fp8,0,0.6059893369674683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,2,128,1,float16,float16,0,0.62281600634257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,12,128,1,float16,fp8,0,0.6458400090535482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,12,128,1,fp8,fp8,0,0.5302506685256958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,2,128,1,fp8,fp8,0,0.4944800138473511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,2,128,1,float16,fp8,0,0.609770655632019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,4,128,1,float16,fp8,0,0.6164640188217163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,4,128,1,float16,float16,0,0.6270453135172526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,12,128,1,float16,fp8,0,0.3470666805903117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,12,4,128,1,fp8,fp8,0,0.5026986598968506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,12,128,1,float16,float16,0,0.35469333330790204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,1,128,1,float16,float16,0,0.3385173479715983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,1,128,1,float16,fp8,0,0.33211199442545575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,1,128,1,fp8,fp8,0,0.2549920082092285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,12,128,1,fp8,fp8,0,0.2779093384742737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,2,128,1,float16,float16,0,0.34112000465393066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,2,128,1,float16,fp8,0,0.33662935098012287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,2,128,1,fp8,fp8,0,0.2558026711146037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,4,128,1,float16,fp8,0,0.33793067932128906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,12,128,1,float16,fp8,0,0.18159999450047812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,12,128,1,float16,float16,0,0.18387732903162637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,4,128,1,fp8,fp8,0,0.2611306707064311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,12,128,1,fp8,fp8,0,0.15467199683189392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,12,4,128,1,float16,float16,0,0.34403733412424725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,1,128,1,float16,fp8,0,0.17123200496037802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,1,128,1,fp8,fp8,0,0.1444480021794637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,2,128,1,float16,float16,0,0.17590934038162231
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,2,128,1,fp8,fp8,0,0.1455466647942861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,1,128,1,float16,float16,0,0.17479999860127768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,2,128,1,float16,fp8,0,0.17216533422470093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,4,128,1,float16,float16,0,0.1770346760749817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,4,128,1,fp8,fp8,0,0.14807466665903726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,12,4,128,1,float16,fp8,0,0.17403733730316162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,1,128,1,float16,float16,0,0.10411733388900757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,12,128,1,float16,fp8,0,0.10674666364987691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,12,128,1,float16,float16,0,0.1079360047976176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,1,128,1,float16,fp8,0,0.10204266508420308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,1,128,1,fp8,fp8,0,0.08599467078844707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,12,128,1,fp8,fp8,0,0.09475732843081157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,2,128,1,float16,fp8,0,0.10335466265678406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,2,128,1,float16,float16,0,0.10416533549626668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,2,128,1,fp8,fp8,0,0.08689066767692566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,4,128,1,float16,float16,0,0.10437867045402527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,4,128,1,float16,fp8,0,0.10404800375302632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,12,4,128,1,fp8,fp8,0,0.08906666437784831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,12,1,128,1,float16,float16,0,2.526672045389811
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,12,1,128,1,float16,fp8,0,2.387418588002523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,12,1,128,1,fp8,fp8,0,2.0580320358276367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,12,2,128,1,float16,float16,0,2.528026739756266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,12,2,128,1,fp8,fp8,0,2.0658079783121743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,12,2,128,1,float16,fp8,0,2.395263989766439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,12,4,128,1,float16,float16,0,2.574885368347168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,1,128,1,float16,float16,0,1.2559413115183513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,1,128,1,float16,fp8,0,1.2049333254496257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,12,4,128,1,fp8,fp8,0,2.1027305920918784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,12,128,1,float16,fp8,0,1.3055626551310222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,12,4,128,1,float16,fp8,0,2.4567893346150718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,1,128,1,fp8,fp8,0,1.030448039372762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,12,128,1,fp8,fp8,0,1.1242612997690837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,12,128,1,float16,float16,0,1.363925298055013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,2,128,1,float16,float16,0,1.2690506776173909
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,2,128,1,float16,fp8,0,1.2077866395314534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,2,128,1,fp8,fp8,0,1.0399946371714275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,4,128,1,float16,fp8,0,1.2256960074106853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,4,128,1,float16,float16,0,1.2935787041982014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,12,4,128,1,fp8,fp8,0,1.0594879786173503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,12,128,1,float16,float16,0,0.6887253125508627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,12,128,1,float16,fp8,0,0.6705226898193359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,1,128,1,float16,float16,0,0.6346240043640137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,1,128,1,float16,fp8,0,0.6173333326975504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,2,128,1,float16,float16,0,0.6381066640218099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,12,128,1,fp8,fp8,0,0.5813226699829102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,1,128,1,fp8,fp8,0,0.5278346538543701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,2,128,1,fp8,fp8,0,0.5344640016555786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,2,128,1,float16,fp8,0,0.6216053167978922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,4,128,1,float16,float16,0,0.6484533150990804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,12,128,1,float16,float16,0,0.3648000160853068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,1,128,1,float16,float16,0,0.34012266000111896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,1,128,1,float16,fp8,0,0.3296853303909302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,4,128,1,float16,fp8,0,0.6303253173828125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,12,128,1,fp8,fp8,0,0.31134400765101117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,12,128,1,float16,fp8,0,0.35785067081451416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,12,4,128,1,fp8,fp8,0,0.5450986623764038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,1,128,1,fp8,fp8,0,0.271994670232137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,2,128,1,float16,fp8,0,0.3332800070444743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,2,128,1,fp8,fp8,0,0.2743946711222331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,4,128,1,float16,float16,0,0.34758933385213214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,4,128,1,fp8,fp8,0,0.2809600035349528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,2,128,1,float16,float16,0,0.3416159947713216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,12,4,128,1,float16,fp8,0,0.33818666140238446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,1,128,1,float16,fp8,0,0.17216000954310098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,1,128,1,float16,float16,0,0.17638399203618368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,12,128,1,float16,fp8,0,0.19271999597549438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,1,128,1,fp8,fp8,0,0.15035200119018555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,12,128,1,fp8,fp8,0,0.16656532883644104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,12,128,1,float16,float16,0,0.1969119906425476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,2,128,1,float16,float16,0,0.17761600017547607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,2,128,1,float16,fp8,0,0.17298666636149088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,2,128,1,fp8,fp8,0,0.15296533703804016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,4,128,1,float16,float16,0,0.18134399255116782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,4,128,1,fp8,fp8,0,0.15533333023389181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,12,128,1,float16,float16,0,0.10759466886520386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,12,4,128,1,float16,fp8,0,0.17762666940689087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,1,128,1,float16,float16,0,0.1002293328444163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,12,128,1,float16,fp8,0,0.10666132966677348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,1,128,1,fp8,fp8,0,0.08493866523106892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,1,128,1,float16,fp8,0,0.09846933682759602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,2,128,1,fp8,fp8,0,0.08749866485595703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,12,128,1,fp8,fp8,0,0.09723732868830363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,2,128,1,float16,float16,0,0.10086400310198466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,2,128,1,float16,fp8,0,0.09942400455474854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,4,128,1,fp8,fp8,0,0.09154666463534038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,4,128,1,float16,fp8,0,0.10146666566530864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,12,128,1,float16,float16,0,0.06676800052324931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,12,4,128,1,float16,float16,0,0.10301867127418518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,1,128,1,float16,float16,0,0.06414400041103363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,1,128,1,float16,fp8,0,0.06330133477846782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,2,128,1,float16,float16,0,0.06438399851322174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,1,128,1,fp8,fp8,0,0.05514133473237356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,12,128,1,float16,fp8,0,0.06559466818968455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,12,128,1,fp8,fp8,0,0.06121600170930227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,2,128,1,float16,fp8,0,0.06317866841952006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,2,128,1,fp8,fp8,0,0.055888002117474876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,4,128,1,float16,fp8,0,0.06379200021425883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,4,128,1,float16,float16,0,0.06417599817117055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,12,4,128,1,fp8,fp8,0,0.0562666654586792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,12,1,128,1,float16,float16,0,1.6006560325622559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,12,1,128,1,float16,fp8,0,1.5427093505859375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,12,1,128,1,fp8,fp8,0,1.3569119771321614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,12,2,128,1,float16,float16,0,1.6198132832845051
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,12,2,128,1,float16,fp8,0,1.555221398671468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,12,2,128,1,fp8,fp8,0,1.3715945879618328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,12,4,128,1,float16,float16,0,1.6566613515218098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,1,128,1,float16,float16,0,0.8095253308614095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,1,128,1,float16,fp8,0,0.782591978708903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,12,128,1,float16,fp8,0,0.8658560117085775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,12,128,1,fp8,fp8,0,0.7753120263417562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,1,128,1,fp8,fp8,0,0.6944053173065186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,12,4,128,1,fp8,fp8,0,1.409663995107015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,12,128,1,float16,float16,0,0.8980159759521484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,12,4,128,1,float16,fp8,0,1.5897013346354167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,2,128,1,float16,float16,0,0.8185599644978842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,2,128,1,fp8,fp8,0,0.7017066478729248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,2,128,1,float16,fp8,0,0.7893706957499186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,12,128,1,float16,float16,0,0.46194132169087726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,4,128,1,float16,float16,0,0.8311306635538737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,4,128,1,float16,fp8,0,0.8093706766764323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,12,4,128,1,fp8,fp8,0,0.7186613082885742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,1,128,1,float16,float16,0,0.41996268431345624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,12,128,1,float16,fp8,0,0.4511359930038452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,1,128,1,float16,fp8,0,0.406879981358846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,1,128,1,fp8,fp8,0,0.36233067512512207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,2,128,1,float16,float16,0,0.4229439894358317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,2,128,1,float16,fp8,0,0.4121333360671997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,12,128,1,fp8,fp8,0,0.40455468495686847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,2,128,1,fp8,fp8,0,0.3662506739298503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,4,128,1,float16,float16,0,0.42983468373616535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,4,128,1,float16,fp8,0,0.42024000485738117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,12,128,1,float16,float16,0,0.24835199117660522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,12,4,128,1,fp8,fp8,0,0.37519999345143634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,1,128,1,float16,float16,0,0.2278826634089152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,12,128,1,fp8,fp8,0,0.21657600005467734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,12,128,1,float16,fp8,0,0.24343466758728027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,1,128,1,float16,fp8,0,0.22060799598693848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,2,128,1,float16,float16,0,0.2300800085067749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,2,128,1,fp8,fp8,0,0.19328000148137411
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,2,128,1,float16,fp8,0,0.22495466470718384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,4,128,1,float16,float16,0,0.23412799835205078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,1,128,1,fp8,fp8,0,0.19005866845448813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,4,128,1,fp8,fp8,0,0.1963520050048828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,12,4,128,1,float16,fp8,0,0.2290453314781189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,12,128,1,float16,float16,0,0.1373973290125529
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,1,128,1,float16,float16,0,0.12156266967455547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,12,128,1,float16,fp8,0,0.13432533542315164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,12,128,1,fp8,fp8,0,0.12138666709264119
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,1,128,1,fp8,fp8,0,0.10812266667683919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,2,128,1,float16,float16,0,0.12244266271591187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,1,128,1,float16,fp8,0,0.11872532963752747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,2,128,1,float16,fp8,0,0.12004799644152324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,2,128,1,fp8,fp8,0,0.11028800408045451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,4,128,1,float16,float16,0,0.12430933117866516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,4,128,1,float16,fp8,0,0.121888001759847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,12,4,128,1,fp8,fp8,0,0.11250666777292888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,1,128,1,float16,float16,0,0.07217599948247273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,12,128,1,float16,float16,0,0.07860800127188365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,1,128,1,float16,fp8,0,0.0707893321911494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,12,128,1,float16,fp8,0,0.07745066781838734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,12,128,1,fp8,fp8,0,0.07400000095367432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,1,128,1,fp8,fp8,0,0.06262933214505513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,2,128,1,float16,float16,0,0.0718560020128886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,2,128,1,float16,fp8,0,0.0711946686108907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,4,128,1,float16,float16,0,0.07307200133800507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,2,128,1,fp8,fp8,0,0.06365333497524261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,4,128,1,fp8,fp8,0,0.06578133503595988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,12,128,1,float16,float16,0,0.05239466826121012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,12,128,1,float16,fp8,0,0.05303466816743215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,12,4,128,1,float16,fp8,0,0.0717493345340093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,1,128,1,float16,float16,0,0.05206400156021118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,1,128,1,float16,fp8,0,0.051674668987592064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,2,128,1,float16,float16,0,0.05193066596984863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,1,128,1,fp8,fp8,0,0.04610133171081543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,12,128,1,fp8,fp8,0,0.047983999053637184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,2,128,1,float16,fp8,0,0.05162133276462555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,4,128,1,float16,float16,0,0.05189866820971171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,4,128,1,fp8,fp8,0,0.046666666865348816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,4,128,1,float16,fp8,0,0.05206400156021118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,12,2,128,1,fp8,fp8,0,0.04598399996757507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,12,1,128,1,float16,fp8,0,1.5022239685058594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,12,1,128,1,float16,float16,0,1.487936019897461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,12,1,128,1,fp8,fp8,0,1.314677317937215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,12,2,128,1,float16,float16,0,1.509610652923584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,12,2,128,1,float16,fp8,0,1.5148693720499675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,12,2,128,1,fp8,fp8,0,1.4074880282084148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,12,4,128,1,float16,float16,0,1.5684107144673665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,1,128,1,float16,float16,0,0.7619893550872803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,12,128,1,float16,float16,0,0.9599359830220541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,1,128,1,float16,fp8,0,0.76201065381368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,12,4,128,1,float16,fp8,0,1.5768639246622722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,12,128,1,fp8,fp8,0,0.8232853412628174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,12,128,1,float16,fp8,0,0.9299039840698242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,12,4,128,1,fp8,fp8,0,1.5141332944234211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,1,128,1,fp8,fp8,0,0.6621119976043701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,2,128,1,float16,float16,0,0.7698187033335367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,2,128,1,float16,fp8,0,0.7709386348724365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,2,128,1,fp8,fp8,0,0.7253493467966715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,4,128,1,float16,float16,0,0.7995680173238119
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,4,128,1,fp8,fp8,0,0.7715306282043457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,12,4,128,1,float16,fp8,0,0.7964586416880289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,1,128,1,float16,float16,0,0.39737598101298016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,12,128,1,float16,float16,0,0.49276800950368244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,1,128,1,float16,fp8,0,0.3976000150044759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,1,128,1,fp8,fp8,0,0.3454933166503906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,12,128,1,float16,fp8,0,0.47551465034484863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,2,128,1,float16,float16,0,0.40298132101694745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,2,128,1,float16,fp8,0,0.4001866579055786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,12,128,1,fp8,fp8,0,0.4231040080388387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,2,128,1,fp8,fp8,0,0.36208001772562665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,4,128,1,float16,float16,0,0.41790934403737384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,4,128,1,float16,fp8,0,0.4121493498484294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,12,128,1,float16,float16,0,0.26030399401982623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,12,4,128,1,fp8,fp8,0,0.3975093364715576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,12,128,1,float16,fp8,0,0.251311997572581
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,1,128,1,float16,float16,0,0.21549866596857706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,1,128,1,float16,fp8,0,0.21290665864944458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,1,128,1,fp8,fp8,0,0.17924267053604126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,2,128,1,float16,fp8,0,0.21417067448298135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,12,128,1,fp8,fp8,0,0.2250666618347168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,2,128,1,float16,float16,0,0.21710399786631265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,2,128,1,fp8,fp8,0,0.18416533867518106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,4,128,1,float16,fp8,0,0.22059732675552368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,4,128,1,fp8,fp8,0,0.20327999194463095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,12,4,128,1,float16,float16,0,0.22530666987101236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,12,128,1,float16,float16,0,0.1532426675160726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,12,128,1,fp8,fp8,0,0.1185706655184428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,12,128,1,float16,fp8,0,0.14847999811172485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,1,128,1,float16,float16,0,0.11547733346621196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,1,128,1,float16,fp8,0,0.11457066734631856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,1,128,1,fp8,fp8,0,0.09900800387064616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,2,128,1,float16,fp8,0,0.11528533697128296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,2,128,1,float16,float16,0,0.11583999792734782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,4,128,1,float16,float16,0,0.12101333340009053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,2,128,1,fp8,fp8,0,0.10107733805974324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,4,128,1,float16,fp8,0,0.1200266679128011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,12,128,1,float16,float16,0,0.07637866834799449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,12,4,128,1,fp8,fp8,0,0.10840533177057902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,12,128,1,float16,fp8,0,0.07473599910736084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,1,128,1,float16,fp8,0,0.06279466549555461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,12,128,1,fp8,fp8,0,0.06901866694291432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,1,128,1,fp8,fp8,0,0.05709333221117655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,2,128,1,float16,fp8,0,0.06533866624037425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,2,128,1,float16,float16,0,0.0653546651204427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,1,128,1,float16,float16,0,0.06326400240262349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,2,128,1,fp8,fp8,0,0.058575997749964394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,4,128,1,float16,float16,0,0.06752533217271169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,4,128,1,fp8,fp8,0,0.06390400230884552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,12,4,128,1,float16,fp8,0,0.06791466474533081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,12,128,1,float16,float16,0,0.04302933315436045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,12,128,1,float16,fp8,0,0.04253333310286204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,1,128,1,float16,float16,0,0.0386613334218661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,12,128,1,fp8,fp8,0,0.040661332507928215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,1,128,1,float16,fp8,0,0.03902933249870936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,1,128,1,fp8,fp8,0,0.03719466676314672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,2,128,1,float16,float16,0,0.039488000174363456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,4,128,1,float16,float16,0,0.03996799886226654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,2,128,1,float16,fp8,0,0.039605334401130676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,4,128,1,fp8,fp8,0,0.039135999977588654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,2,128,1,fp8,fp8,0,0.03801066676775614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,12,4,128,1,float16,fp8,0,0.04015466570854187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,12,128,1,float16,float16,0,0.03679466744263967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,1,128,1,float16,float16,0,0.03498133271932602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,12,128,1,fp8,fp8,0,0.03178133318821589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,12,128,1,float16,fp8,0,0.03659733384847641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,1,128,1,float16,fp8,0,0.03495466709136963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,1,128,1,fp8,fp8,0,0.031221332649389904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,2,128,1,float16,fp8,0,0.03565866748491923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,2,128,1,fp8,fp8,0,0.03121600051720937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,4,128,1,float16,float16,0,0.03557866563399633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,4,128,1,fp8,fp8,0,0.0322826678554217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,2,128,1,float16,float16,0,0.03475733349720637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,12,4,128,1,float16,fp8,0,0.03581333408753077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,12,1,128,1,float16,float16,0,1.160522699356079
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,12,1,128,1,fp8,fp8,0,1.0975573062896729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,12,1,128,1,float16,fp8,0,1.1574026743570964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,12,2,128,1,float16,float16,0,1.1813653310139973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,12,2,128,1,float16,fp8,0,1.1740000247955322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,12,2,128,1,fp8,fp8,0,1.169104019800822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,12,4,128,1,float16,float16,0,1.2626240253448486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,1,128,1,float16,float16,0,0.5916159947713217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,1,128,1,float16,fp8,0,0.5881866614023844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,12,128,1,float16,float16,0,0.794426679611206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,12,4,128,1,float16,fp8,0,1.2753973007202148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,12,4,128,1,fp8,fp8,0,1.281008005142212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,12,128,1,fp8,fp8,0,0.7072693506876627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,12,128,1,float16,fp8,0,0.7618666489919027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,1,128,1,fp8,fp8,0,0.5501866738001505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,2,128,1,float16,float16,0,0.603440006573995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,2,128,1,float16,fp8,0,0.6003199815750122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,2,128,1,fp8,fp8,0,0.598688006401062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,4,128,1,float16,float16,0,0.6377280155817667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,4,128,1,float16,fp8,0,0.6279573440551758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,12,4,128,1,fp8,fp8,0,0.6479093233744303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,12,128,1,float16,float16,0,0.4143093427022298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,1,128,1,float16,float16,0,0.30937065680821735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,1,128,1,float16,fp8,0,0.3101973334948222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,12,128,1,float16,fp8,0,0.39044801394144696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,1,128,1,fp8,fp8,0,0.28547199567159015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,12,128,1,fp8,fp8,0,0.3632426659266154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,2,128,1,float16,float16,0,0.31567466259002686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,2,128,1,fp8,fp8,0,0.3086400032043457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,2,128,1,float16,fp8,0,0.31250667572021484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,4,128,1,float16,fp8,0,0.32612266143163043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,4,128,1,fp8,fp8,0,0.3354826768239339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,12,4,128,1,float16,float16,0,0.33444801966349286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,1,128,1,float16,float16,0,0.16726400454839072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,12,128,1,float16,float16,0,0.21689599752426147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,12,128,1,float16,fp8,0,0.20983999967575073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,12,128,1,fp8,fp8,0,0.19244800011316934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,1,128,1,fp8,fp8,0,0.15376533071200052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,2,128,1,float16,float16,0,0.171450674533844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,1,128,1,float16,fp8,0,0.1676373283068339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,2,128,1,fp8,fp8,0,0.158053328593572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,2,128,1,float16,fp8,0,0.1699733336766561
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,4,128,1,float16,float16,0,0.17944000164667764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,4,128,1,float16,fp8,0,0.1783413290977478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,12,128,1,float16,float16,0,0.1360373298327128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,12,4,128,1,fp8,fp8,0,0.17371733983357748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,1,128,1,float16,fp8,0,0.0937013328075409
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,1,128,1,float16,float16,0,0.09383466839790344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,12,128,1,float16,fp8,0,0.12979732950528464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,12,128,1,fp8,fp8,0,0.1029919981956482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,1,128,1,fp8,fp8,0,0.08549333612124126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,2,128,1,float16,float16,0,0.09578133622805278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,2,128,1,fp8,fp8,0,0.08813333511352539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,2,128,1,float16,fp8,0,0.09539199868837993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,4,128,1,float16,fp8,0,0.09827733039855957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,4,128,1,fp8,fp8,0,0.0937493344148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,12,4,128,1,float16,float16,0,0.1004960040251414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,12,128,1,float16,float16,0,0.06768533090750377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,12,128,1,float16,fp8,0,0.0647573322057724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,1,128,1,float16,float16,0,0.05342933535575867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,1,128,1,fp8,fp8,0,0.04905599852403005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,2,128,1,float16,fp8,0,0.055018668373425804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,1,128,1,float16,fp8,0,0.05272533496220907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,12,128,1,fp8,fp8,0,0.061834668119748436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,2,128,1,float16,float16,0,0.055104002356529236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,2,128,1,fp8,fp8,0,0.052186667919158936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,4,128,1,float16,float16,0,0.05789333085219065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,4,128,1,fp8,fp8,0,0.0554666668176651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,12,4,128,1,float16,fp8,0,0.058090666929880776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,12,128,1,float16,float16,0,0.0391893337170283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,12,128,1,float16,fp8,0,0.03818666686614355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,12,128,1,fp8,fp8,0,0.037530665596326195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,1,128,1,float16,float16,0,0.03435733417669932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,2,128,1,fp8,fp8,0,0.033941333492596946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,2,128,1,float16,float16,0,0.035402665535608925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,1,128,1,float16,fp8,0,0.03499733408292135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,2,128,1,float16,fp8,0,0.03477866699298223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,4,128,1,float16,fp8,0,0.0364479993780454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,4,128,1,float16,float16,0,0.03580799947182337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,1,128,1,fp8,fp8,0,0.03321066747109095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,12,4,128,1,fp8,fp8,0,0.03541333228349686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,12,128,1,float16,float16,0,0.03009066730737686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,12,128,1,float16,fp8,0,0.03012799968322118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,1,128,1,float16,float16,0,0.02827200045188268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,1,128,1,fp8,fp8,0,0.026144000391165417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,1,128,1,float16,fp8,0,0.027653334041436512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,12,128,1,fp8,fp8,0,0.027935999135176342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,2,128,1,float16,fp8,0,0.02810666710138321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,2,128,1,float16,float16,0,0.028512001037597656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,2,128,1,fp8,fp8,0,0.026485333840052288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,4,128,1,float16,float16,0,0.029120000700155895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,4,128,1,float16,fp8,0,0.029130667448043823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,12,4,128,1,fp8,fp8,0,0.02754133443037669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,12,128,1,float16,fp8,0,0.024874667326609295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,1,128,1,float16,float16,0,0.024832000335057575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,1,128,1,float16,fp8,0,0.025258667767047882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,12,128,1,fp8,fp8,0,0.02369066576162974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,12,128,1,float16,float16,0,0.02481599897146225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,1,128,1,fp8,fp8,0,0.02342933416366577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,2,128,1,float16,fp8,0,0.02516266703605652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,2,128,1,float16,float16,0,0.024864000578721363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,4,128,1,float16,float16,0,0.025519999365011852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,2,128,1,fp8,fp8,0,0.02332799881696701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,4,128,1,float16,fp8,0,0.025263999899228413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,12,4,128,1,fp8,fp8,0,0.023434666295846302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,12,1,128,1,float16,float16,0,0.5060106515884399
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,12,1,128,1,float16,fp8,0,0.5045546690622965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,12,1,128,1,fp8,fp8,0,0.4907413323720296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,12,2,128,1,float16,fp8,0,0.5199306805928549
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,12,2,128,1,float16,float16,0,0.5209066470464071
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,12,2,128,1,fp8,fp8,0,0.5224639972050985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,12,4,128,1,float16,float16,0,0.5548586845397949
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,1,128,1,float16,float16,0,0.2654186685880025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,12,4,128,1,float16,fp8,0,0.5473173459370931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,12,128,1,float16,float16,0,0.3734826644261678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,12,128,1,float16,fp8,0,0.3551786740620931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,12,4,128,1,fp8,fp8,0,0.5771146615346273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,12,128,1,fp8,fp8,0,0.3308853308359782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,1,128,1,fp8,fp8,0,0.24729599555333456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,1,128,1,float16,fp8,0,0.2640960017840068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,2,128,1,float16,float16,0,0.2730613350868225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,2,128,1,fp8,fp8,0,0.2654186685880025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,2,128,1,float16,fp8,0,0.2699573238690694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,4,128,1,float16,float16,0,0.29016000032424927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,4,128,1,float16,fp8,0,0.28506133953730267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,12,4,128,1,fp8,fp8,0,0.29850133260091144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,12,128,1,float16,float16,0,0.19873599211374918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,1,128,1,float16,fp8,0,0.14331199725468954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,1,128,1,float16,float16,0,0.14312533537546793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,1,128,1,fp8,fp8,0,0.1372053325176239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,12,128,1,fp8,fp8,0,0.17522132396697998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,12,128,1,float16,fp8,0,0.19129067659378052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,2,128,1,float16,float16,0,0.14815466602643332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,2,128,1,fp8,fp8,0,0.14283200105031332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,2,128,1,float16,fp8,0,0.1454026699066162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,4,128,1,float16,fp8,0,0.15546666582425436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,4,128,1,float16,float16,0,0.15756266315778097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,12,4,128,1,fp8,fp8,0,0.1602826714515686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,12,128,1,float16,float16,0,0.1221386690934499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,12,128,1,float16,fp8,0,0.11789333820343018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,12,128,1,fp8,fp8,0,0.09603200356165568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,1,128,1,float16,fp8,0,0.08322666585445404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,1,128,1,fp8,fp8,0,0.07707199951012929
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,2,128,1,float16,float16,0,0.08582933743794759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,2,128,1,float16,fp8,0,0.08520533641179402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,2,128,1,fp8,fp8,0,0.08064533273379008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,1,128,1,float16,float16,0,0.08281599978605907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,4,128,1,float16,float16,0,0.09132799506187439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,4,128,1,float16,fp8,0,0.09066667159398396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,12,128,1,float16,float16,0,0.0633653352657954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,12,128,1,float16,fp8,0,0.061199997862180076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,12,4,128,1,fp8,fp8,0,0.0867039958635966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,12,128,1,fp8,fp8,0,0.05715199808279673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,1,128,1,float16,float16,0,0.04844266672929128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,1,128,1,float16,fp8,0,0.04850666721661886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,2,128,1,float16,float16,0,0.05036266644795736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,1,128,1,fp8,fp8,0,0.044405331214269005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,4,128,1,float16,float16,0,0.052928000688552856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,2,128,1,fp8,fp8,0,0.046351999044418335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,2,128,1,float16,fp8,0,0.05049066742261251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,4,128,1,fp8,fp8,0,0.05036266644795736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,12,128,1,float16,float16,0,0.037258667250474296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,12,128,1,float16,fp8,0,0.03665599972009659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,12,128,1,fp8,fp8,0,0.034703999757766724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,1,128,1,float16,fp8,0,0.033071999748547874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,1,128,1,float16,float16,0,0.03309333324432373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,12,4,128,1,float16,fp8,0,0.05266133447488149
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,1,128,1,fp8,fp8,0,0.030810666580994923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,2,128,1,float16,float16,0,0.03445333242416382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,2,128,1,float16,fp8,0,0.034058667719364166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,2,128,1,fp8,fp8,0,0.03091199944416682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,4,128,1,float16,float16,0,0.03570133447647095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,12,128,1,float16,float16,0,0.027119999130566914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,4,128,1,float16,fp8,0,0.03540800015131632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,12,128,1,float16,fp8,0,0.02714666724205017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,12,128,1,fp8,fp8,0,0.025114665428797405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,12,4,128,1,fp8,fp8,0,0.0330079992612203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,1,128,1,float16,float16,0,0.02460266649723053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,1,128,1,float16,fp8,0,0.02535466601451238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,1,128,1,fp8,fp8,0,0.02364266663789749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,2,128,1,float16,fp8,0,0.026000000536441803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,2,128,1,fp8,fp8,0,0.023733332753181458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,4,128,1,float16,fp8,0,0.02608533451954524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,2,128,1,float16,float16,0,0.025093334416548412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,4,128,1,float16,float16,0,0.02584533393383026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,12,128,1,float16,float16,0,0.022128000855445862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,12,4,128,1,fp8,fp8,0,0.024853333830833435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,12,128,1,float16,fp8,0,0.022554665803909302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,12,128,1,fp8,fp8,0,0.020394666741291683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,1,128,1,float16,float16,0,0.021664001047611237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,2,128,1,float16,float16,0,0.021888000269730885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,2,128,1,float16,fp8,0,0.022197333474953968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,2,128,1,fp8,fp8,0,0.019989332805077236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,1,128,1,fp8,fp8,0,0.020410666863123577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,1,128,1,float16,fp8,0,0.021482666333516438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,4,128,1,float16,float16,0,0.021589333812395733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,4,128,1,float16,fp8,0,0.02199999988079071
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,12,4,128,1,fp8,fp8,0,0.020768000433842342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,12,128,1,float16,float16,0,0.021130666136741638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,12,128,1,float16,fp8,0,0.021312000850836437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,12,128,1,fp8,fp8,0,0.019509332875410717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,1,128,1,float16,fp8,0,0.0210506667693456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,1,128,1,fp8,fp8,0,0.019765333582957584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,1,128,1,float16,float16,0,0.02094399929046631
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,2,128,1,fp8,fp8,0,0.019194666296243668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,2,128,1,float16,float16,0,0.02063999945918719
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,2,128,1,float16,fp8,0,0.020970667401949566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,4,128,1,float16,float16,0,0.020538666596015293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,4,128,1,float16,fp8,0,0.021546666820844013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,12,4,128,1,fp8,fp8,0,0.019674666225910187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,12,1,128,1,float16,fp8,0,0.24940800666809082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,12,1,128,1,fp8,fp8,0,0.24749332666397095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,12,2,128,1,float16,float16,0,0.25657065709431964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,12,1,128,1,float16,float16,0,0.24971733490626016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,12,2,128,1,float16,fp8,0,0.2537386616071065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,12,4,128,1,float16,float16,0,0.2735946575800578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,12,2,128,1,fp8,fp8,0,0.27313599983851117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,12,128,1,float16,float16,0,0.20559465885162354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,12,4,128,1,fp8,fp8,0,0.3007253408432007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,12,128,1,fp8,fp8,0,0.17012266318003336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,12,4,128,1,float16,fp8,0,0.26970134178797406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,1,128,1,float16,fp8,0,0.13831999897956848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,12,128,1,float16,fp8,0,0.19896000623703003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,1,128,1,float16,float16,0,0.13849600156148276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,1,128,1,fp8,fp8,0,0.13573333621025085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,2,128,1,float16,float16,0,0.14125333229700723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,2,128,1,fp8,fp8,0,0.14569600423177084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,2,128,1,float16,fp8,0,0.14034133156140646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,4,128,1,float16,float16,0,0.15225066741307577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,4,128,1,float16,fp8,0,0.15026133259137472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,12,128,1,float16,float16,0,0.11311466495196025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,12,4,128,1,fp8,fp8,0,0.15588266650835672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,12,128,1,float16,fp8,0,0.10955199599266052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,12,128,1,fp8,fp8,0,0.0960053304831187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,1,128,1,float16,float16,0,0.07841066519419353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,2,128,1,float16,float16,0,0.08152533570925395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,1,128,1,float16,fp8,0,0.07945066690444946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,2,128,1,fp8,fp8,0,0.08027733365694682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,2,128,1,float16,fp8,0,0.08063466846942902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,1,128,1,fp8,fp8,0,0.0768746683994929
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,4,128,1,float16,float16,0,0.08737599849700928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,4,128,1,fp8,fp8,0,0.08657067020734151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,12,128,1,float16,fp8,0,0.05964266757170359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,12,4,128,1,float16,fp8,0,0.08641599615414937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,1,128,1,float16,fp8,0,0.04643733302752177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,12,128,1,float16,float16,0,0.061941335598627724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,1,128,1,float16,float16,0,0.04634666442871094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,12,128,1,fp8,fp8,0,0.05629333357016245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,1,128,1,fp8,fp8,0,0.04438399771849314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,2,128,1,float16,float16,0,0.047482664386431374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,2,128,1,fp8,fp8,0,0.046112000942230225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,4,128,1,float16,float16,0,0.050341332952181496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,4,128,1,float16,fp8,0,0.049829334020614624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,12,128,1,float16,float16,0,0.036474667489528656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,2,128,1,float16,fp8,0,0.04656533400217692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,12,128,1,float16,fp8,0,0.03530666728814443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,12,128,1,fp8,fp8,0,0.03461866577466329
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,12,4,128,1,fp8,fp8,0,0.050245334704717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,1,128,1,fp8,fp8,0,0.030554667115211487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,2,128,1,float16,fp8,0,0.033088001112143196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,2,128,1,fp8,fp8,0,0.030640001098314922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,2,128,1,float16,float16,0,0.03346666693687439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,1,128,1,float16,fp8,0,0.03309866786003113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,1,128,1,float16,float16,0,0.03275199979543686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,4,128,1,float16,float16,0,0.03707200040419897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,4,128,1,float16,fp8,0,0.03400533397992452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,12,128,1,float16,float16,0,0.02613866577545802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,12,4,128,1,fp8,fp8,0,0.032746667663256325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,12,128,1,float16,fp8,0,0.025786665578683216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,1,128,1,fp8,fp8,0,0.023205332458019257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,1,128,1,float16,float16,0,0.024266667664051056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,12,128,1,fp8,fp8,0,0.02480533222357432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,2,128,1,float16,float16,0,0.024480000138282776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,2,128,1,float16,fp8,0,0.024453334510326385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,1,128,1,float16,fp8,0,0.024725332856178284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,4,128,1,float16,float16,0,0.024821333587169647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,2,128,1,fp8,fp8,0,0.023013333479563396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,12,128,1,float16,fp8,0,0.02063999945918719
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,12,128,1,float16,float16,0,0.020794666061798733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,4,128,1,fp8,fp8,0,0.02497066557407379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,12,4,128,1,float16,fp8,0,0.02595199892918269
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,1,128,1,float16,float16,0,0.020303999384244282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,12,128,1,fp8,fp8,0,0.020501332978407543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,1,128,1,fp8,fp8,0,0.02022933339079221
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,2,128,1,float16,float16,0,0.020202666521072388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,4,128,1,float16,float16,0,0.02045866722861926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,1,128,1,float16,fp8,0,0.01995733380317688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,2,128,1,float16,fp8,0,0.020058666666348774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,2,128,1,fp8,fp8,0,0.02042666698495547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,4,128,1,fp8,fp8,0,0.020661332954963047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,12,4,128,1,float16,fp8,0,0.020245333512624104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,12,128,1,float16,fp8,0,0.020213333268960316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,12,128,1,float16,float16,0,0.01969066634774208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,1,128,1,fp8,fp8,0,0.019530666371186573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,2,128,1,float16,fp8,0,0.019978666057189304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,2,128,1,float16,float16,0,0.019152000546455383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,12,128,1,fp8,fp8,0,0.019509332875410717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,2,128,1,fp8,fp8,0,0.01956266661485036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,1,128,1,float16,float16,0,0.019120000302791595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,1,128,1,float16,fp8,0,0.020058666666348774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,4,128,1,float16,float16,0,0.019546666493018467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,4,128,1,float16,fp8,0,0.01995733380317688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,12,128,1,float16,fp8,0,0.019061333189407986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,12,4,128,1,fp8,fp8,0,0.019695999721686046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,1,128,1,float16,fp8,0,0.019354666272799175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,12,128,1,float16,float16,0,0.018496000518401463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,12,128,1,fp8,fp8,0,0.018687999496857326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,1,128,1,fp8,fp8,0,0.01899733394384384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,1,128,1,float16,float16,0,0.019285333653291065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,2,128,1,float16,float16,0,0.019007999449968338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,2,128,1,fp8,fp8,0,0.018789333601792652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,4,128,1,fp8,fp8,0,0.019007999449968338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,2,128,1,float16,fp8,0,0.01915733392039935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,4,128,1,float16,float16,0,0.01865600049495697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,12,4,128,1,float16,fp8,0,0.019152000546455383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,12,1,128,1,float16,fp8,0,0.16356799999872842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,12,1,128,1,float16,float16,0,0.16481600205103555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,12,1,128,1,fp8,fp8,0,0.17641599973042807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,12,2,128,1,float16,float16,0,0.1671839952468872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,12,2,128,1,float16,fp8,0,0.16571199893951416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,12,2,128,1,fp8,fp8,0,0.18245333433151245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,12,4,128,1,float16,fp8,0,0.17501866817474365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,12,4,128,1,float16,float16,0,0.1786186695098877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,12,128,1,float16,float16,0,0.11083733042081197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,12,128,1,float16,fp8,0,0.10732266306877136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,12,4,128,1,fp8,fp8,0,0.19839467604955038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,1,128,1,float16,float16,0,0.08909866213798523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,12,128,1,fp8,fp8,0,0.11505066355069478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,1,128,1,float16,fp8,0,0.08975999553998311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,2,128,1,float16,float16,0,0.09203733007113139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,1,128,1,fp8,fp8,0,0.09769066174825032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,2,128,1,float16,fp8,0,0.09201600154240926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,2,128,1,fp8,fp8,0,0.10034132997194926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,4,128,1,fp8,fp8,0,0.10689066847165425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,4,128,1,float16,float16,0,0.09875733653704326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,12,4,128,1,float16,fp8,0,0.09694400429725647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,12,128,1,float16,float16,0,0.06247466802597046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,12,128,1,fp8,fp8,0,0.06693333387374878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,1,128,1,float16,float16,0,0.05153599878152212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,1,128,1,float16,fp8,0,0.05166399975617727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,12,128,1,float16,fp8,0,0.06091733276844025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,2,128,1,float16,fp8,0,0.05258666475613912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,2,128,1,float16,float16,0,0.05207466582457224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,1,128,1,fp8,fp8,0,0.054458667834599815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,4,128,1,float16,fp8,0,0.0554613322019577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,2,128,1,fp8,fp8,0,0.056847999493281044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,4,128,1,float16,float16,0,0.05562133093674978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,12,128,1,float16,float16,0,0.03495466709136963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,12,128,1,float16,fp8,0,0.03442666679620743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,12,4,128,1,fp8,fp8,0,0.061306665341059365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,12,128,1,fp8,fp8,0,0.03733866661787033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,1,128,1,float16,float16,0,0.03314133236805598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,1,128,1,float16,fp8,0,0.03327466547489166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,2,128,1,float16,float16,0,0.03350399931271871
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,1,128,1,fp8,fp8,0,0.03461866577466329
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,2,128,1,float16,fp8,0,0.03335466732581457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,4,128,1,float16,fp8,0,0.03459733227888743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,12,128,1,float16,float16,0,0.02741333345572154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,4,128,1,fp8,fp8,0,0.0366293340921402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,2,128,1,fp8,fp8,0,0.0349440003434817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,12,128,1,float16,fp8,0,0.02752533306678136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,12,4,128,1,float16,float16,0,0.03411199897527695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,1,128,1,float16,fp8,0,0.026261332134405773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,1,128,1,float16,float16,0,0.02624000112215678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,12,128,1,fp8,fp8,0,0.028565332293510437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,1,128,1,fp8,fp8,0,0.02699200063943863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,2,128,1,float16,fp8,0,0.026949333647886913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,2,128,1,float16,float16,0,0.026496000587940216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,4,128,1,float16,float16,0,0.026975999275843304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,2,128,1,fp8,fp8,0,0.027386667827765148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,4,128,1,float16,fp8,0,0.027263998985290527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,12,128,1,float16,float16,0,0.019914666811625164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,12,4,128,1,fp8,fp8,0,0.02863999952872594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,1,128,1,float16,float16,0,0.019199999670187633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,12,128,1,float16,fp8,0,0.020560000091791153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,12,128,1,fp8,fp8,0,0.020351999749739964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,1,128,1,float16,fp8,0,0.019354666272799175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,1,128,1,fp8,fp8,0,0.01985599969824155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,2,128,1,float16,float16,0,0.019013332823912304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,2,128,1,float16,fp8,0,0.019424000134070713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,2,128,1,fp8,fp8,0,0.020346666375796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,4,128,1,float16,fp8,0,0.019466667125622433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,4,128,1,float16,float16,0,0.0194560003777345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,12,4,128,1,fp8,fp8,0,0.020432000358899433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,12,128,1,float16,fp8,0,0.018735999862353008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,12,128,1,float16,float16,0,0.01838933303952217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,12,128,1,fp8,fp8,0,0.019391999890406925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,1,128,1,float16,float16,0,0.018464000274737675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,2,128,1,float16,fp8,0,0.019002666076024372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,1,128,1,fp8,fp8,0,0.019487999379634857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,2,128,1,float16,float16,0,0.018330667167901993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,2,128,1,fp8,fp8,0,0.01931200052301089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,1,128,1,float16,fp8,0,0.01858666663368543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,4,128,1,float16,float16,0,0.018496000518401463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,4,128,1,float16,fp8,0,0.019007999449968338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,12,128,1,fp8,fp8,0,0.018698666244745255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,12,4,128,1,fp8,fp8,0,0.019306667149066925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,12,128,1,float16,float16,0,0.01706133286158244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,12,128,1,float16,fp8,0,0.01798933371901512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,2,128,1,float16,float16,0,0.01812800019979477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,1,128,1,float16,fp8,0,0.018197332819302876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,2,128,1,float16,fp8,0,0.018474667022625606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,1,128,1,fp8,fp8,0,0.01884799947341283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,1,128,1,float16,float16,0,0.01766933376590411
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,4,128,1,float16,float16,0,0.017877332866191864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,2,128,1,fp8,fp8,0,0.0191040001809597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,4,128,1,float16,fp8,0,0.018394666413466137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,12,128,1,float16,float16,0,0.01735466718673706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,12,128,1,float16,fp8,0,0.017583999782800674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,12,4,128,1,fp8,fp8,0,0.01913600042462349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,1,128,1,fp8,fp8,0,0.018794666975736618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,1,128,1,float16,float16,0,0.01736533393462499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,1,128,1,float16,fp8,0,0.018053332964579265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,12,128,1,fp8,fp8,0,0.018496000518401463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,2,128,1,float16,float16,0,0.01801066721479098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,2,128,1,float16,fp8,0,0.01800000046690305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,4,128,1,float16,float16,0,0.017530667285124462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,2,128,1,fp8,fp8,0,0.018661333868900936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,4,128,1,float16,fp8,0,0.017637333522240322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,12,4,128,1,fp8,fp8,0,0.019023999571800232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,12,1,128,1,float16,float16,0,0.11150399843851726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,12,1,128,1,float16,fp8,0,0.11170666416486104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,12,1,128,1,fp8,fp8,0,0.14426133036613464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,12,2,128,1,float16,float16,0,0.11473066608111064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,12,2,128,1,float16,fp8,0,0.11470400293668111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,12,2,128,1,fp8,fp8,0,0.14680533607800803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,12,4,128,1,float16,float16,0,0.12319999933242798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,12,128,1,float16,float16,0,0.07453866799672444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,12,4,128,1,float16,fp8,0,0.12024000287055969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,12,4,128,1,fp8,fp8,0,0.1540000041325887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,12,128,1,float16,fp8,0,0.07225066423416138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,1,128,1,float16,fp8,0,0.062261333068211876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,12,128,1,fp8,fp8,0,0.08956799904505412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,1,128,1,float16,float16,0,0.06182399888833364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,1,128,1,fp8,fp8,0,0.07849066456158955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,2,128,1,float16,fp8,0,0.06394133468468984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,2,128,1,fp8,fp8,0,0.08258133133252461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,2,128,1,float16,float16,0,0.06300266583760579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,4,128,1,float16,float16,0,0.06735999882221222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,4,128,1,float16,fp8,0,0.06651733318964641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,12,128,1,float16,float16,0,0.040031999349594116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,12,128,1,float16,fp8,0,0.03921599934498469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,1,128,1,float16,float16,0,0.037578667203585304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,12,4,128,1,fp8,fp8,0,0.08509866396586101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,12,128,1,fp8,fp8,0,0.049285332361857094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,1,128,1,float16,fp8,0,0.038362666964530945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,1,128,1,fp8,fp8,0,0.04613866905371348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,2,128,1,float16,float16,0,0.038176000118255615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,2,128,1,float16,fp8,0,0.03864533454179764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,4,128,1,float16,fp8,0,0.039093332986036934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,2,128,1,fp8,fp8,0,0.04682666560014089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,4,128,1,float16,float16,0,0.03896533449490865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,12,4,128,1,fp8,fp8,0,0.04874666531880697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,12,128,1,float16,float16,0,0.028016000986099243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,12,128,1,float16,fp8,0,0.02826133370399475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,12,128,1,fp8,fp8,0,0.032933334509531655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,1,128,1,float16,float16,0,0.027301333844661713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,1,128,1,fp8,fp8,0,0.03159466634194056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,2,128,1,float16,float16,0,0.027402666707833607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,1,128,1,float16,fp8,0,0.027162666122118633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,2,128,1,fp8,fp8,0,0.032058666149775185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,2,128,1,float16,fp8,0,0.027679999669392902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,4,128,1,float16,float16,0,0.028346667687098186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,4,128,1,float16,fp8,0,0.028501334289709728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,1,128,1,fp8,fp8,0,0.024138666689395905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,12,4,128,1,fp8,fp8,0,0.03293866664171219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,12,128,1,float16,float16,0,0.02143999934196472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,1,128,1,float16,fp8,0,0.021738665799299877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,1,128,1,float16,float16,0,0.021322667598724365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,12,128,1,float16,fp8,0,0.02144533395767212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,12,128,1,fp8,fp8,0,0.02462399999300639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,2,128,1,float16,float16,0,0.021216000119845074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,4,128,1,float16,float16,0,0.02164799968401591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,2,128,1,float16,fp8,0,0.021642667551835377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,2,128,1,fp8,fp8,0,0.024117333193620045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,4,128,1,float16,fp8,0,0.022341333329677582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,12,128,1,float16,fp8,0,0.01798933371901512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,12,128,1,fp8,fp8,0,0.01974933346112569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,12,4,128,1,fp8,fp8,0,0.024085332949956257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,1,128,1,float16,float16,0,0.017727999637524288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,1,128,1,float16,fp8,0,0.018090666582187016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,1,128,1,fp8,fp8,0,0.01930133377512296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,2,128,1,float16,float16,0,0.017701332767804463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,12,128,1,float16,float16,0,0.018181333939234417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,2,128,1,float16,fp8,0,0.017717332889636356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,4,128,1,float16,float16,0,0.017583999782800674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,12,128,1,float16,float16,0,0.017152000218629837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,4,128,1,float16,fp8,0,0.018207999567190807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,4,128,1,fp8,fp8,0,0.019546666493018467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,12,2,128,1,fp8,fp8,0,0.01937066639463107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,12,128,1,float16,fp8,0,0.01736533393462499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,1,128,1,float16,fp8,0,0.017664000391960144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,1,128,1,float16,float16,0,0.01670933390657107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,12,128,1,fp8,fp8,0,0.018592000007629395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,1,128,1,fp8,fp8,0,0.019199999670187633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,2,128,1,float16,float16,0,0.01687466725707054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,4,128,1,float16,float16,0,0.01725333308180173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,2,128,1,fp8,fp8,0,0.018613333503405254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,2,128,1,float16,fp8,0,0.017797333498795826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,4,128,1,float16,fp8,0,0.017557332913080852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,12,4,128,1,fp8,fp8,0,0.018538666268189747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,12,128,1,fp8,fp8,0,0.018426666657129925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,12,128,1,float16,fp8,0,0.016800000021855038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,12,128,1,float16,float16,0,0.016917333006858826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,2,128,1,float16,fp8,0,0.017488000293572743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,1,128,1,fp8,fp8,0,0.018415999909241993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,2,128,1,float16,float16,0,0.017152000218629837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,1,128,1,float16,fp8,0,0.01735466718673706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,1,128,1,float16,float16,0,0.016789333273967106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,2,128,1,fp8,fp8,0,0.018565333137909572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,4,128,1,float16,fp8,0,0.017360000560681026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,4,128,1,float16,float16,0,0.017173333714405697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,12,4,128,1,fp8,fp8,0,0.018559999763965607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,12,128,1,float16,float16,0,0.016613333175579708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,1,128,1,float16,fp8,0,0.017029333859682083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,1,128,1,float16,float16,0,0.016858667135238647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,1,128,1,fp8,fp8,0,0.01851733277241389
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,2,128,1,fp8,fp8,0,0.01825599993268649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,12,128,1,float16,fp8,0,0.016879999389251072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,12,128,1,fp8,fp8,0,0.01834133391578992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,2,128,1,float16,fp8,0,0.017610666652520496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,2,128,1,float16,float16,0,0.01670933390657107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,4,128,1,float16,float16,0,0.016773333152135212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,4,128,1,fp8,fp8,0,0.01868266612291336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,12,4,128,1,float16,fp8,0,0.01692266638080279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,12,1,128,1,float16,fp8,0,0.09495466947555542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,12,1,128,1,fp8,fp8,0,0.12723199526468912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,12,1,128,1,float16,float16,0,0.09437333544095357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,12,2,128,1,float16,float16,0,0.09583466251691182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,12,2,128,1,float16,fp8,0,0.09512000282605489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,12,2,128,1,fp8,fp8,0,0.12874666849772134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,12,4,128,1,float16,float16,0,0.09943999846776326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,12,4,128,1,float16,fp8,0,0.09873599807421367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,12,4,128,1,fp8,fp8,0,0.13172266880671182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,12,128,1,float16,fp8,0,0.05385066568851471
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,12,128,1,float16,float16,0,0.05387733379999796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,1,128,1,float16,float16,0,0.05426133175690969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,1,128,1,float16,fp8,0,0.05417599777380625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,12,128,1,fp8,fp8,0,0.07520000139872234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,1,128,1,fp8,fp8,0,0.07076266904671986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,2,128,1,float16,float16,0,0.05492266515890757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,2,128,1,float16,fp8,0,0.05486399928728739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,4,128,1,float16,float16,0,0.055344000458717346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,2,128,1,fp8,fp8,0,0.07094933092594147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,4,128,1,float16,fp8,0,0.05574400226275126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,12,4,128,1,fp8,fp8,0,0.07214400172233582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,12,128,1,float16,fp8,0,0.034645333886146545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,1,128,1,float16,float16,0,0.03554133325815201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,12,128,1,float16,float16,0,0.03507733345031738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,12,128,1,fp8,fp8,0,0.044437333941459656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,1,128,1,float16,fp8,0,0.035232000052928925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,2,128,1,float16,fp8,0,0.03563733398914337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,2,128,1,float16,float16,0,0.035205334424972534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,1,128,1,fp8,fp8,0,0.04429866870244344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,2,128,1,fp8,fp8,0,0.044362664222717285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,4,128,1,float16,float16,0,0.03630933413902918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,4,128,1,fp8,fp8,0,0.04531733194986979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,12,4,128,1,float16,fp8,0,0.03648533423741659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,12,128,1,float16,float16,0,0.02405333270629247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,12,128,1,fp8,fp8,0,0.028965334097544353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,1,128,1,float16,float16,0,0.02407466620206833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,1,128,1,float16,fp8,0,0.024234667420387268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,12,128,1,float16,fp8,0,0.023984000086784363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,2,128,1,float16,fp8,0,0.02481599897146225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,1,128,1,fp8,fp8,0,0.02847466617822647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,2,128,1,float16,float16,0,0.02418133368094762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,4,128,1,float16,float16,0,0.024357333779335022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,4,128,1,float16,fp8,0,0.024458666642506916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,2,128,1,fp8,fp8,0,0.02902399996916453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,12,4,128,1,fp8,fp8,0,0.02845333268245061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,12,128,1,float16,float16,0,0.020453333854675293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,12,128,1,fp8,fp8,0,0.023546665906906128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,1,128,1,float16,float16,0,0.020245333512624104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,12,128,1,float16,fp8,0,0.02027200038234393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,1,128,1,float16,fp8,0,0.020799999435742695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,1,128,1,fp8,fp8,0,0.02364266663789749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,2,128,1,float16,fp8,0,0.02086399992307027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,2,128,1,float16,float16,0,0.02025066688656807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,4,128,1,float16,float16,0,0.0206986665725708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,2,128,1,fp8,fp8,0,0.02327999969323476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,4,128,1,fp8,fp8,0,0.023775999744733173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,12,4,128,1,float16,fp8,0,0.02080533280968666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,12,128,1,float16,float16,0,0.017231999586025875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,12,128,1,float16,fp8,0,0.017968000223239262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,12,128,1,fp8,fp8,0,0.019109333554903667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,1,128,1,float16,float16,0,0.017045332739750545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,1,128,1,float16,fp8,0,0.01748266691962878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,1,128,1,fp8,fp8,0,0.018922666708628338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,2,128,1,fp8,fp8,0,0.019141333798567455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,4,128,1,float16,fp8,0,0.017610666652520496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,2,128,1,float16,fp8,0,0.01730666682124138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,4,128,1,float16,float16,0,0.017210666090250015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,2,128,1,float16,float16,0,0.01725333308180173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,12,128,1,float16,float16,0,0.016751999656359356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,12,4,128,1,fp8,fp8,0,0.019023999571800232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,12,128,1,float16,fp8,0,0.0169813334941864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,12,128,1,fp8,fp8,0,0.018346666047970455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,1,128,1,float16,fp8,0,0.017173333714405697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,2,128,1,float16,float16,0,0.01710933322707812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,1,128,1,float16,float16,0,0.016976000120242436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,2,128,1,float16,fp8,0,0.017504000415404636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,1,128,1,fp8,fp8,0,0.018618666877349217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,4,128,1,float16,float16,0,0.01710933322707812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,2,128,1,fp8,fp8,0,0.018250666558742523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,4,128,1,fp8,fp8,0,0.018432000031073887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,12,4,128,1,float16,fp8,0,0.017583999782800674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,12,128,1,float16,fp8,0,0.016544000556071598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,12,128,1,float16,float16,0,0.0164533331990242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,12,128,1,fp8,fp8,0,0.018042666216691334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,1,128,1,float16,float16,0,0.016890666137139004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,1,128,1,float16,fp8,0,0.017077332983414333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,1,128,1,fp8,fp8,0,0.018357332795858383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,2,128,1,float16,fp8,0,0.016757333030303318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,2,128,1,float16,float16,0,0.016656000167131424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,2,128,1,fp8,fp8,0,0.018725333114465077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,4,128,1,float16,fp8,0,0.017317333569129307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,4,128,1,fp8,fp8,0,0.018453333526849747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,12,4,128,1,float16,float16,0,0.016303999970356624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,12,128,1,float16,float16,0,0.01616000011563301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,12,128,1,float16,fp8,0,0.0161013330022494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,12,128,1,fp8,fp8,0,0.018085333208243053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,1,128,1,float16,fp8,0,0.016783999900023144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,1,128,1,float16,float16,0,0.01672533278663953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,1,128,1,fp8,fp8,0,0.018794666975736618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,2,128,1,float16,fp8,0,0.016837333639462788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,2,128,1,fp8,fp8,0,0.018383999665578205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,2,128,1,float16,float16,0,0.01647466669480006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,4,128,1,float16,float16,0,0.016314666718244553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,4,128,1,float16,fp8,0,0.01647466669480006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,12,4,128,1,fp8,fp8,0,0.0189280000825723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,8,1,128,1,float16,float16,0,8.379818598429361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,8,1,128,1,float16,fp8,0,8.397685368855795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,8,2,128,1,float16,fp8,0,8.464885075887045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,8,2,128,1,fp8,fp8,0,5.439680099487305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,8,1,128,1,fp8,fp8,0,5.483829498291016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,8,2,128,1,float16,float16,0,8.387450536092123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,8,4,128,1,float16,float16,0,8.46557362874349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,1,128,1,float16,float16,0,4.146976153055827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,8,128,1,float16,fp8,0,4.020880063374837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,8,128,1,float16,float16,0,4.035738627115886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,1,128,1,float16,fp8,0,3.9817492167154946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,8,4,128,1,float16,fp8,0,8.341818491617838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,8,128,1,fp8,fp8,0,2.753024101257324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,1,128,1,fp8,fp8,0,2.7245171864827475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,8,4,128,1,fp8,fp8,0,5.437536239624023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,2,128,1,float16,float16,0,4.032314618428548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,2,128,1,float16,fp8,0,4.137621243794759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,2,128,1,fp8,fp8,0,2.738117218017578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,8,128,1,float16,float16,0,2.069162686665853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,4,128,1,float16,float16,0,4.013669331868489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,4,128,1,float16,fp8,0,4.264357248942058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,8,4,128,1,fp8,fp8,0,2.752197265625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,1,128,1,float16,float16,0,2.0935519536336265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,8,128,1,float16,fp8,0,2.0704426765441895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,1,128,1,float16,fp8,0,2.075573285420736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,1,128,1,fp8,fp8,0,1.43614927927653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,8,128,1,fp8,fp8,0,1.4435307184855144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,2,128,1,float16,float16,0,2.132853349049886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,2,128,1,float16,fp8,0,2.074399948120117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,2,128,1,fp8,fp8,0,1.444101333618164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,8,128,1,float16,float16,0,1.1285706361134846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,4,128,1,float16,float16,0,2.110223929087321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,4,128,1,float16,fp8,0,2.0866506894429526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,8,4,128,1,fp8,fp8,0,1.452287991841634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,8,128,1,float16,fp8,0,1.1421653429667156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,1,128,1,float16,float16,0,1.147274653116862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,8,128,1,fp8,fp8,0,0.7205706437428793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,1,128,1,fp8,fp8,0,0.7201226552327474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,1,128,1,float16,fp8,0,1.137610673904419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,2,128,1,float16,float16,0,1.1493866443634033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,2,128,1,float16,fp8,0,1.1391626993815105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,2,128,1,fp8,fp8,0,0.7252693176269531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,4,128,1,float16,float16,0,1.1538399855295818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,4,128,1,float16,fp8,0,1.1422186692555745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,8,4,128,1,fp8,fp8,0,0.7277759710947672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,8,1,128,1,float16,float16,0,4.661781311035156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,8,1,128,1,float16,fp8,0,4.555551846822103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,8,1,128,1,fp8,fp8,0,3.16046937306722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,8,2,128,1,float16,float16,0,4.760261217753093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,8,2,128,1,float16,fp8,0,4.561221440633138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,8,2,128,1,fp8,fp8,0,3.172069231669108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,8,4,128,1,float16,float16,0,4.733274777730306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,8,128,1,float16,float16,0,2.339893341064453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,1,128,1,float16,float16,0,2.357429345448812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,8,4,128,1,fp8,fp8,0,3.2014452616373696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,1,128,1,float16,fp8,0,2.357840061187744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,8,128,1,float16,fp8,0,2.384607950846354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,8,4,128,1,float16,fp8,0,4.8358720143636065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,8,128,1,fp8,fp8,0,1.6631147066752117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,1,128,1,fp8,fp8,0,1.6417120297749836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,2,128,1,float16,float16,0,2.357978661855062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,2,128,1,float16,fp8,0,2.344693342844645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,2,128,1,fp8,fp8,0,1.6528746287027996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,4,128,1,fp8,fp8,0,1.6662453015645344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,4,128,1,float16,float16,0,2.4222346941630044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,8,4,128,1,float16,fp8,0,2.3568639755249023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,8,128,1,float16,float16,0,1.2404533227284749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,1,128,1,float16,float16,0,1.258138656616211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,1,128,1,float16,fp8,0,1.2460532983144124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,1,128,1,fp8,fp8,0,0.8208373387654623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,8,128,1,float16,fp8,0,1.234239975611369
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,8,128,1,fp8,fp8,0,0.8336106936136881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,2,128,1,float16,float16,0,1.256170670191447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,2,128,1,float16,fp8,0,1.2465706666310628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,2,128,1,fp8,fp8,0,0.8245279788970947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,4,128,1,float16,float16,0,1.2665653228759766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,8,128,1,float16,fp8,0,0.6013546784718832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,8,128,1,float16,float16,0,0.6067253351211548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,4,128,1,fp8,fp8,0,0.8345866998036703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,8,4,128,1,float16,fp8,0,1.271941343943278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,1,128,1,float16,float16,0,0.6092160145441691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,8,128,1,fp8,fp8,0,0.44598933060963947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,1,128,1,float16,fp8,0,0.5992266734441122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,1,128,1,fp8,fp8,0,0.44067200024922687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,2,128,1,float16,fp8,0,0.6007680098215739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,2,128,1,fp8,fp8,0,0.44257601102193195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,2,128,1,float16,float16,0,0.6090773344039917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,4,128,1,float16,float16,0,0.6156906684239706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,4,128,1,float16,fp8,0,0.6057493289311727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,8,4,128,1,fp8,fp8,0,0.4474560022354126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,8,1,128,1,float16,float16,0,3.2914454142252603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,8,1,128,1,fp8,fp8,0,2.2972853978474936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,8,1,128,1,float16,fp8,0,3.3377386728922525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,8,2,128,1,float16,float16,0,3.286208152770996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,8,2,128,1,float16,fp8,0,3.273226737976074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,8,2,128,1,fp8,fp8,0,2.3083573977152505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,8,4,128,1,float16,float16,0,3.3666559855143228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,1,128,1,float16,float16,0,1.6924907366434734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,8,128,1,float16,float16,0,1.690773328145345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,8,128,1,float16,fp8,0,1.6846720377604167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,8,128,1,fp8,fp8,0,1.2256266276041667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,8,4,128,1,fp8,fp8,0,2.3413707415262857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,8,4,128,1,float16,fp8,0,3.288112004597982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,1,128,1,float16,fp8,0,1.6829226811726887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,1,128,1,fp8,fp8,0,1.199407974878947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,2,128,1,float16,float16,0,1.6982986132303874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,2,128,1,float16,fp8,0,1.6856320699055989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,2,128,1,fp8,fp8,0,1.2083146572113037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,4,128,1,fp8,fp8,0,1.22325332959493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,4,128,1,float16,fp8,0,1.6983359654744465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,8,128,1,float16,float16,0,0.9067946275075277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,8,4,128,1,float16,float16,0,1.7407840092976887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,1,128,1,float16,float16,0,0.9170506795247396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,8,128,1,float16,fp8,0,0.900544007619222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,8,128,1,fp8,fp8,0,0.6144426663716634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,1,128,1,float16,fp8,0,0.9078346888224283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,1,128,1,fp8,fp8,0,0.6000320116678873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,2,128,1,float16,float16,0,0.9166719913482666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,2,128,1,float16,fp8,0,0.9211893081665039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,2,128,1,fp8,fp8,0,0.6066986719767252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,4,128,1,float16,float16,0,0.9195199807484945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,4,128,1,float16,fp8,0,0.9154720306396484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,8,128,1,float16,float16,0,0.4488639831542969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,1,128,1,float16,float16,0,0.45260266462961835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,8,4,128,1,fp8,fp8,0,0.6147306760152181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,8,128,1,fp8,fp8,0,0.3370026747385661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,8,128,1,float16,fp8,0,0.44441600640614826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,1,128,1,fp8,fp8,0,0.33246399958928424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,1,128,1,float16,fp8,0,0.4455626805623372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,2,128,1,float16,fp8,0,0.4486773411432902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,2,128,1,float16,float16,0,0.45154666900634766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,2,128,1,fp8,fp8,0,0.3332586685816447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,4,128,1,float16,float16,0,0.45635199546813965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,4,128,1,float16,fp8,0,0.45002134641011554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,8,4,128,1,fp8,fp8,0,0.33752532800038654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,8,1,128,1,float16,float16,0,4.339573224385579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,8,1,128,1,fp8,fp8,0,3.044069290161133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,8,1,128,1,float16,fp8,0,4.422618548075358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,8,2,128,1,float16,float16,0,4.486405372619629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,8,2,128,1,fp8,fp8,0,3.062368075052897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,8,2,128,1,float16,fp8,0,4.3462826410929365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,8,4,128,1,float16,float16,0,4.353103955586751
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,1,128,1,float16,float16,0,2.2334133783976235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,8,128,1,float16,float16,0,2.2249013582865396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,1,128,1,float16,fp8,0,2.167850653330485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,8,4,128,1,float16,fp8,0,4.467472076416016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,8,128,1,fp8,fp8,0,1.6058665911356609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,8,128,1,float16,fp8,0,2.195295969645182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,8,4,128,1,fp8,fp8,0,3.101626714070638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,1,128,1,fp8,fp8,0,1.568021297454834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,2,128,1,float16,float16,0,2.210261344909668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,2,128,1,float16,fp8,0,2.1770346959431968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,2,128,1,fp8,fp8,0,1.5780107180277507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,8,128,1,float16,float16,0,1.1706773440043132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,4,128,1,float16,fp8,0,2.199834664662679
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,4,128,1,float16,float16,0,2.2119040489196777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,8,4,128,1,fp8,fp8,0,1.5949494043986003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,1,128,1,float16,float16,0,1.1440107027689617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,8,128,1,float16,fp8,0,1.1459999879201253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,8,128,1,fp8,fp8,0,0.8501333395640055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,1,128,1,fp8,fp8,0,0.8257546424865723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,2,128,1,float16,float16,0,1.1464373270670574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,1,128,1,float16,fp8,0,1.1469226678212483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,2,128,1,float16,fp8,0,1.136080026626587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,2,128,1,fp8,fp8,0,0.8321920235951742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,4,128,1,float16,float16,0,1.1599626541137695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,4,128,1,float16,fp8,0,1.1453226407368977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,8,128,1,float16,float16,0,0.6256906588872274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,1,128,1,float16,float16,0,0.6282773415247599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,8,128,1,float16,fp8,0,0.6303253173828125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,8,128,1,fp8,fp8,0,0.42797334988911945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,8,4,128,1,fp8,fp8,0,0.8434453010559082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,1,128,1,float16,fp8,0,0.6237653493881226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,1,128,1,fp8,fp8,0,0.4178080161412557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,2,128,1,float16,float16,0,0.6311253309249878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,2,128,1,float16,fp8,0,0.6247413158416748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,2,128,1,fp8,fp8,0,0.4200799862543742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,4,128,1,float16,float16,0,0.6355306704839071
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,4,128,1,float16,fp8,0,0.6309920152028402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,8,128,1,float16,float16,0,0.3054293394088745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,8,4,128,1,fp8,fp8,0,0.4268480141957601
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,1,128,1,float16,float16,0,0.3036693334579468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,8,128,1,float16,fp8,0,0.3019520044326782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,1,128,1,float16,fp8,0,0.2997866670290629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,1,128,1,fp8,fp8,0,0.23092265923817953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,8,128,1,fp8,fp8,0,0.23663467168807983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,2,128,1,float16,float16,0,0.30484267075856525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,2,128,1,float16,fp8,0,0.30083199342091876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,2,128,1,fp8,fp8,0,0.23364800214767456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,4,128,1,float16,float16,0,0.3059840003649394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,4,128,1,float16,fp8,0,0.3020319938659668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,8,4,128,1,fp8,fp8,0,0.23616000016530356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,8,1,128,1,fp8,fp8,0,1.8806932767232258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,8,1,128,1,float16,fp8,0,2.535802682240804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,8,1,128,1,float16,float16,0,2.5706613858540854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,8,2,128,1,float16,fp8,0,2.5473012924194336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,8,2,128,1,fp8,fp8,0,1.8948373794555664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,8,2,128,1,float16,float16,0,2.6331574122111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,8,4,128,1,float16,float16,0,2.626688003540039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,1,128,1,float16,float16,0,1.340559959411621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,8,128,1,float16,float16,0,1.3431199391682942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,1,128,1,float16,fp8,0,1.3071040312449138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,8,4,128,1,fp8,fp8,0,1.9232373237609863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,8,4,128,1,float16,fp8,0,2.5922932624816895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,8,128,1,float16,fp8,0,1.3256693681081135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,8,128,1,fp8,fp8,0,1.0091573397318523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,1,128,1,fp8,fp8,0,0.9754933516184489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,2,128,1,float16,float16,0,1.3293493588765461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,2,128,1,float16,fp8,0,1.314079999923706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,2,128,1,fp8,fp8,0,0.984602689743042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,4,128,1,float16,float16,0,1.3472320238749187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,4,128,1,float16,fp8,0,1.322544018427531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,8,4,128,1,fp8,fp8,0,1.00053866704305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,1,128,1,float16,float16,0,0.7039039929707845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,8,128,1,float16,fp8,0,0.7018346786499023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,8,128,1,fp8,fp8,0,0.516053318977356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,1,128,1,float16,fp8,0,0.6957066853841146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,1,128,1,fp8,fp8,0,0.49012800057729083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,2,128,1,float16,float16,0,0.7058986822764078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,8,128,1,float16,float16,0,0.7206079959869385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,2,128,1,float16,fp8,0,0.6986613273620605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,2,128,1,fp8,fp8,0,0.49586133162180585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,4,128,1,float16,fp8,0,0.7062186400095621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,4,128,1,float16,float16,0,0.7132426897684733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,8,4,128,1,fp8,fp8,0,0.5078879992167155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,8,128,1,float16,float16,0,0.351855993270874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,8,128,1,fp8,fp8,0,0.2772586743036906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,1,128,1,float16,float16,0,0.3449653387069702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,8,128,1,float16,fp8,0,0.3528053363164266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,1,128,1,float16,fp8,0,0.34039998054504395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,1,128,1,fp8,fp8,0,0.26686932643254596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,2,128,1,float16,float16,0,0.3456266721089681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,2,128,1,float16,fp8,0,0.3422986666361491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,2,128,1,fp8,fp8,0,0.26926400264104206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,4,128,1,fp8,fp8,0,0.2730506658554077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,8,128,1,float16,float16,0,0.20522665977478027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,4,128,1,float16,fp8,0,0.3490026791890462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,1,128,1,float16,float16,0,0.2000853419303894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,8,4,128,1,float16,float16,0,0.35155200958251953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,8,128,1,float16,fp8,0,0.2016213337580363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,8,128,1,fp8,fp8,0,0.16237333416938782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,1,128,1,float16,fp8,0,0.19802133242289224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,1,128,1,fp8,fp8,0,0.1551040013631185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,2,128,1,float16,fp8,0,0.1995733380317688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,4,128,1,float16,float16,0,0.20350933074951172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,4,128,1,float16,fp8,0,0.20219733317693075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,2,128,1,float16,float16,0,0.20200000206629434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,2,128,1,fp8,fp8,0,0.15685866276423135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,8,4,128,1,fp8,fp8,0,0.16090666254361471
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,8,1,128,1,float16,float16,0,2.5640320777893066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,8,1,128,1,float16,fp8,0,2.4955946604410806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,8,1,128,1,fp8,fp8,0,1.9367893536885579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,8,2,128,1,float16,float16,0,2.5778187115987143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,8,2,128,1,float16,fp8,0,2.5477919578552246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,8,2,128,1,fp8,fp8,0,1.950442632039388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,8,4,128,1,float16,float16,0,2.639941374460856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,1,128,1,float16,float16,0,1.2915093104044597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,1,128,1,float16,fp8,0,1.2707946300506592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,8,128,1,float16,float16,0,1.3337173461914062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,8,128,1,float16,fp8,0,1.3133653004964192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,8,4,128,1,fp8,fp8,0,1.989525318145752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,8,4,128,1,float16,fp8,0,2.5745387077331543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,8,128,1,fp8,fp8,0,1.0417439937591553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,1,128,1,fp8,fp8,0,0.9897279739379883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,2,128,1,float16,float16,0,1.3047093550364177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,2,128,1,float16,fp8,0,1.2775200208028157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,2,128,1,fp8,fp8,0,0.9978079795837402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,4,128,1,float16,float16,0,1.3181173006693523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,4,128,1,float16,fp8,0,1.2977973620096843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,8,128,1,float16,float16,0,0.6924853324890137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,1,128,1,float16,float16,0,0.672602653503418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,8,128,1,float16,fp8,0,0.6868639787038168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,8,4,128,1,fp8,fp8,0,1.0204266707102458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,8,128,1,fp8,fp8,0,0.5486826499303182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,1,128,1,float16,fp8,0,0.6606719891230265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,1,128,1,fp8,fp8,0,0.5195093154907227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,2,128,1,float16,float16,0,0.6768373648325602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,2,128,1,float16,fp8,0,0.6655626694361368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,2,128,1,fp8,fp8,0,0.5237226486206055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,4,128,1,float16,float16,0,0.6908640066782633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,4,128,1,float16,fp8,0,0.6746880213419596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,8,4,128,1,fp8,fp8,0,0.5360053380330404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,1,128,1,float16,float16,0,0.3689333200454712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,8,128,1,float16,float16,0,0.37850133577982586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,8,128,1,float16,fp8,0,0.37324265638987225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,8,128,1,fp8,fp8,0,0.2839946746826172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,1,128,1,fp8,fp8,0,0.2654186685880025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,1,128,1,float16,fp8,0,0.36402666568756104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,2,128,1,float16,float16,0,0.37231465180714923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,2,128,1,fp8,fp8,0,0.2700960040092468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,2,128,1,float16,fp8,0,0.3667946656545003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,4,128,1,float16,float16,0,0.37582401434580487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,1,128,1,float16,float16,0,0.18126400311787924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,4,128,1,float16,fp8,0,0.3707946538925171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,8,4,128,1,fp8,fp8,0,0.2754559914271037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,8,128,1,float16,fp8,0,0.18661334117253622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,8,128,1,fp8,fp8,0,0.15714133779207864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,8,128,1,float16,float16,0,0.18996800978978476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,1,128,1,float16,fp8,0,0.17787732680638632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,1,128,1,fp8,fp8,0,0.1473973294099172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,2,128,1,float16,float16,0,0.18293333053588867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,2,128,1,fp8,fp8,0,0.1495680014292399
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,4,128,1,float16,float16,0,0.18707732359568277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,4,128,1,float16,fp8,0,0.18357332547505698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,4,128,1,fp8,fp8,0,0.15251200397809347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,8,2,128,1,float16,fp8,0,0.17910399039586386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,8,128,1,float16,float16,0,0.10752532879511516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,8,128,1,float16,fp8,0,0.1065013309319814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,1,128,1,float16,float16,0,0.1043893297513326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,1,128,1,float16,fp8,0,0.10307733217875163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,8,128,1,fp8,fp8,0,0.09299733241399129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,1,128,1,fp8,fp8,0,0.08588266372680664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,2,128,1,float16,float16,0,0.10421866178512573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,2,128,1,float16,fp8,0,0.1032319962978363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,2,128,1,fp8,fp8,0,0.08628267049789429
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,4,128,1,fp8,fp8,0,0.08922666311264038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,4,128,1,float16,float16,0,0.10607999563217163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,8,4,128,1,float16,fp8,0,0.10362666845321655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,8,1,128,1,float16,float16,0,1.5910560290018718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,8,1,128,1,float16,fp8,0,1.5460586547851562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,8,1,128,1,fp8,fp8,0,1.2523146470387776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,8,2,128,1,float16,float16,0,1.6082293192545574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,8,2,128,1,fp8,fp8,0,1.2700586318969727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,8,2,128,1,float16,fp8,0,1.5621280670166016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,8,4,128,1,float16,float16,0,1.660373369852702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,1,128,1,float16,float16,0,0.8112746874491373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,8,128,1,float16,float16,0,0.8531839847564697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,8,4,128,1,float16,fp8,0,1.587125301361084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,1,128,1,float16,fp8,0,0.796074628829956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,8,128,1,float16,fp8,0,0.8331253528594971
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,8,128,1,fp8,fp8,0,0.6862506866455078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,8,4,128,1,fp8,fp8,0,1.2929600079854329
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,1,128,1,fp8,fp8,0,0.6450826724370321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,2,128,1,float16,float16,0,0.8196586767832438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,2,128,1,float16,fp8,0,0.7989386717478434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,2,128,1,fp8,fp8,0,0.6510986487070719
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,4,128,1,float16,float16,0,0.830725351969401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,8,128,1,float16,float16,0,0.449343999226888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,4,128,1,float16,fp8,0,0.8127626578013102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,8,4,128,1,fp8,fp8,0,0.6668960253397623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,8,128,1,float16,fp8,0,0.44061867396036786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,8,128,1,fp8,fp8,0,0.3562613328297933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,1,128,1,float16,float16,0,0.43004266421000165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,1,128,1,fp8,fp8,0,0.32736533880233765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,1,128,1,float16,fp8,0,0.4194186528523763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,2,128,1,float16,float16,0,0.4320160150527954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,2,128,1,float16,fp8,0,0.4236319859822591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,2,128,1,fp8,fp8,0,0.33082133531570435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,4,128,1,float16,float16,0,0.43911465009053546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,8,128,1,float16,float16,0,0.22949333985646567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,8,128,1,float16,fp8,0,0.22674665848414102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,4,128,1,fp8,fp8,0,0.34299735228220624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,8,4,128,1,float16,fp8,0,0.43230934937795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,8,128,1,fp8,fp8,0,0.19213332732518515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,1,128,1,float16,float16,0,0.215338667233785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,1,128,1,float16,fp8,0,0.21136534214019775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,1,128,1,fp8,fp8,0,0.17739200592041016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,2,128,1,float16,fp8,0,0.21368533372879028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,2,128,1,float16,float16,0,0.21784534056981406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,2,128,1,fp8,fp8,0,0.18017599980036417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,4,128,1,float16,fp8,0,0.219866673151652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,4,128,1,float16,float16,0,0.2217173377672831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,8,128,1,float16,float16,0,0.12894399960835776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,1,128,1,float16,float16,0,0.1218933363755544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,8,4,128,1,fp8,fp8,0,0.18396800756454468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,8,128,1,float16,fp8,0,0.1272693375746409
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,8,128,1,fp8,fp8,0,0.11069867014884949
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,1,128,1,float16,fp8,0,0.11975466211636861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,1,128,1,fp8,fp8,0,0.10006399949391682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,2,128,1,float16,float16,0,0.12321066856384277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,2,128,1,fp8,fp8,0,0.1039466659228007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,2,128,1,float16,fp8,0,0.1216373344262441
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,4,128,1,float16,float16,0,0.12548266847928366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,4,128,1,float16,fp8,0,0.12436800201733907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,8,4,128,1,fp8,fp8,0,0.10771200060844421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,8,128,1,float16,float16,0,0.08002666632334392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,8,128,1,float16,fp8,0,0.0788800021012624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,8,128,1,fp8,fp8,0,0.06890133519967397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,1,128,1,float16,float16,0,0.07826666533946991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,1,128,1,float16,fp8,0,0.07720000048478444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,1,128,1,fp8,fp8,0,0.06470400094985962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,2,128,1,float16,float16,0,0.0786186655362447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,2,128,1,float16,fp8,0,0.07764266431331635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,2,128,1,fp8,fp8,0,0.06446399788061778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,4,128,1,float16,float16,0,0.07830399771531422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,4,128,1,float16,fp8,0,0.07793599863847096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,8,4,128,1,fp8,fp8,0,0.06560533245404561
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,8,1,128,1,float16,fp8,0,1.6392265955607097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,8,1,128,1,fp8,fp8,0,1.411861260732015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,8,1,128,1,float16,float16,0,1.7153813044230144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,8,2,128,1,float16,fp8,0,1.6560746828715007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,8,2,128,1,float16,float16,0,1.7319679260253906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,8,2,128,1,fp8,fp8,0,1.4219199816385906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,8,4,128,1,float16,float16,0,1.7730186780293782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,1,128,1,float16,float16,0,0.8651519616444906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,8,128,1,float16,float16,0,0.9278506437937418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,1,128,1,float16,fp8,0,0.8334986368815104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,8,128,1,float16,fp8,0,0.8886026541392008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,8,4,128,1,float16,fp8,0,1.6974933942159016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,8,4,128,1,fp8,fp8,0,1.4545119603474934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,8,128,1,fp8,fp8,0,0.7693440119425455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,1,128,1,fp8,fp8,0,0.7131573359171549
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,2,128,1,float16,float16,0,0.869434674580892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,2,128,1,float16,fp8,0,0.8416746457417806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,2,128,1,fp8,fp8,0,0.7181440194447836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,8,128,1,float16,float16,0,0.476367990175883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,4,128,1,float16,float16,0,0.8912800153096517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,4,128,1,float16,fp8,0,0.8558613459269205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,8,4,128,1,fp8,fp8,0,0.7376853624979655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,1,128,1,float16,float16,0,0.4386986494064331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,8,128,1,float16,fp8,0,0.46270398298899335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,1,128,1,float16,fp8,0,0.42898666858673096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,8,128,1,fp8,fp8,0,0.4026933511098226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,1,128,1,fp8,fp8,0,0.3689279953638713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,2,128,1,float16,float16,0,0.44378666083017987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,2,128,1,float16,fp8,0,0.4320853153864543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,2,128,1,fp8,fp8,0,0.3744639952977498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,4,128,1,float16,float16,0,0.454367995262146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,8,128,1,float16,float16,0,0.2561226685841878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,1,128,1,float16,float16,0,0.23825599749883017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,8,128,1,float16,fp8,0,0.2491626739501953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,4,128,1,fp8,fp8,0,0.3858559926350911
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,8,128,1,fp8,fp8,0,0.20972265799840292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,8,4,128,1,float16,fp8,0,0.4432533184687297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,1,128,1,float16,fp8,0,0.23425066471099854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,1,128,1,fp8,fp8,0,0.1897546648979187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,2,128,1,fp8,fp8,0,0.19383466243743896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,2,128,1,float16,fp8,0,0.23533866802851358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,2,128,1,float16,float16,0,0.2405653397242228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,4,128,1,float16,float16,0,0.24610666433970133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,4,128,1,float16,fp8,0,0.24170132478078207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,8,4,128,1,fp8,fp8,0,0.19919466972351074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,8,128,1,float16,float16,0,0.13250133395195007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,8,128,1,float16,fp8,0,0.12941333651542664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,8,128,1,fp8,fp8,0,0.11753066380818684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,1,128,1,float16,float16,0,0.12062933047612508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,1,128,1,float16,fp8,0,0.11825600266456604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,2,128,1,float16,fp8,0,0.12091199556986491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,2,128,1,float16,float16,0,0.12281599640846252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,2,128,1,fp8,fp8,0,0.10896000266075134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,1,128,1,fp8,fp8,0,0.10619200269381206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,4,128,1,float16,float16,0,0.12502933541933695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,4,128,1,fp8,fp8,0,0.11195733149846394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,8,128,1,float16,float16,0,0.07489599784215291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,8,128,1,float16,fp8,0,0.0745119998852412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,8,128,1,fp8,fp8,0,0.07040533423423767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,8,4,128,1,float16,fp8,0,0.1232319970925649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,1,128,1,float16,fp8,0,0.06870933373769124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,1,128,1,fp8,fp8,0,0.060864001512527466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,1,128,1,float16,float16,0,0.06900266806284587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,2,128,1,float16,float16,0,0.06977066894372304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,2,128,1,fp8,fp8,0,0.061797335743904114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,2,128,1,float16,fp8,0,0.06879466772079468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,4,128,1,fp8,fp8,0,0.06464000046253204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,4,128,1,float16,fp8,0,0.07064533233642578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,8,128,1,float16,fp8,0,0.057631999254226685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,8,4,128,1,float16,float16,0,0.07162133355935414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,1,128,1,float16,float16,0,0.057162667314211525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,8,128,1,fp8,fp8,0,0.05036266644795736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,8,128,1,float16,float16,0,0.05831466615200043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,1,128,1,fp8,fp8,0,0.04913066824277242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,1,128,1,float16,fp8,0,0.05691733459631602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,2,128,1,float16,float16,0,0.05761066575845083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,2,128,1,float16,fp8,0,0.05700799822807312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,2,128,1,fp8,fp8,0,0.04854399959246317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,4,128,1,float16,float16,0,0.05807999769846598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,4,128,1,float16,fp8,0,0.05715199808279673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,8,4,128,1,fp8,fp8,0,0.04955733319123586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,8,1,128,1,float16,fp8,0,1.0419413248697917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,8,1,128,1,float16,float16,0,1.1057066917419434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,8,1,128,1,fp8,fp8,0,0.943008025487264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,8,2,128,1,float16,float16,0,1.1224586963653564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,8,2,128,1,float16,fp8,0,1.053114652633667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,8,2,128,1,fp8,fp8,0,0.9514613151550293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,8,4,128,1,float16,float16,0,1.151253302892049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,1,128,1,float16,float16,0,0.5627413193384806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,8,128,1,float16,float16,0,0.6131999890009562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,1,128,1,float16,fp8,0,0.5364640156428019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,8,128,1,float16,fp8,0,0.5867146650950114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,8,4,128,1,fp8,fp8,0,0.981333335240682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,8,128,1,fp8,fp8,0,0.5330453316370646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,8,4,128,1,float16,fp8,0,1.0780746936798096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,1,128,1,fp8,fp8,0,0.48231999079386395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,2,128,1,float16,float16,0,0.5713226795196533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,2,128,1,float16,fp8,0,0.5416373411814371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,2,128,1,fp8,fp8,0,0.4910026788711548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,4,128,1,float16,float16,0,0.586079994837443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,4,128,1,float16,fp8,0,0.554149349530538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,8,4,128,1,fp8,fp8,0,0.5052213271458944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,1,128,1,float16,float16,0,0.29410133759180707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,8,128,1,float16,float16,0,0.3213599920272827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,1,128,1,float16,fp8,0,0.2851253350575765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,1,128,1,fp8,fp8,0,0.2462666630744934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,8,128,1,float16,fp8,0,0.30990399916966754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,8,128,1,fp8,fp8,0,0.2869973381360372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,2,128,1,float16,float16,0,0.2980479995409648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,2,128,1,float16,fp8,0,0.2897493243217468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,2,128,1,fp8,fp8,0,0.2510400017102559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,4,128,1,float16,float16,0,0.304421325524648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,4,128,1,float16,fp8,0,0.295413335164388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,8,128,1,float16,fp8,0,0.17297599713007608
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,8,128,1,float16,float16,0,0.17512534062067667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,8,4,128,1,fp8,fp8,0,0.26023467381795246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,8,128,1,fp8,fp8,0,0.14924800395965576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,1,128,1,float16,float16,0,0.1516426702340444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,1,128,1,float16,fp8,0,0.14827733238538107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,1,128,1,fp8,fp8,0,0.13458133737246195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,2,128,1,float16,float16,0,0.1548426647981008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,2,128,1,float16,fp8,0,0.15094400445620218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,2,128,1,fp8,fp8,0,0.13612266381581625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,4,128,1,float16,float16,0,0.1604266663392385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,8,128,1,float16,float16,0,0.09249599774678548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,4,128,1,fp8,fp8,0,0.14084266622861227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,1,128,1,float16,float16,0,0.08276266853014629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,8,128,1,fp8,fp8,0,0.08738666772842407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,8,128,1,float16,fp8,0,0.09215999643007915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,1,128,1,float16,fp8,0,0.08210666477680206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,8,4,128,1,float16,fp8,0,0.1560426652431488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,1,128,1,fp8,fp8,0,0.075354665517807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,2,128,1,float16,float16,0,0.08458667000134786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,2,128,1,fp8,fp8,0,0.07826133569081624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,2,128,1,float16,fp8,0,0.08356266220410664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,4,128,1,float16,float16,0,0.08840533097585042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,4,128,1,float16,fp8,0,0.08641599615414937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,8,128,1,float16,float16,0,0.0547680010398229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,8,4,128,1,fp8,fp8,0,0.08297599852085114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,8,128,1,float16,fp8,0,0.054272000988324486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,1,128,1,float16,float16,0,0.05178666611512502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,8,128,1,fp8,fp8,0,0.055200000603993736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,1,128,1,float16,fp8,0,0.05121066669623057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,2,128,1,float16,float16,0,0.05266666909058889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,1,128,1,fp8,fp8,0,0.0469813346862793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,2,128,1,float16,fp8,0,0.05141866703828176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,2,128,1,fp8,fp8,0,0.04757333298524221
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,4,128,1,fp8,fp8,0,0.049141332507133484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,8,128,1,float16,float16,0,0.04747733473777771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,4,128,1,float16,fp8,0,0.052629331747690834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,8,128,1,float16,fp8,0,0.047482664386431374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,8,128,1,fp8,fp8,0,0.041690667470296226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,1,128,1,float16,float16,0,0.04721599817276001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,8,4,128,1,float16,float16,0,0.05247466762860616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,1,128,1,float16,fp8,0,0.04665599763393402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,1,128,1,fp8,fp8,0,0.040735999743143715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,2,128,1,float16,float16,0,0.047242666284243263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,2,128,1,float16,fp8,0,0.04710400104522705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,2,128,1,fp8,fp8,0,0.04095466683308283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,4,128,1,float16,fp8,0,0.04709866642951965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,4,128,1,float16,float16,0,0.04747733473777771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,8,4,128,1,fp8,fp8,0,0.04123199979464213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,8,1,128,1,float16,float16,0,1.0130933125813801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,8,1,128,1,float16,fp8,0,1.0073920090993245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,8,1,128,1,fp8,fp8,0,0.8860639731089274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,8,2,128,1,float16,fp8,0,1.0265973409016926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,8,2,128,1,float16,float16,0,1.0330666700998943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,8,2,128,1,fp8,fp8,0,0.9426133632659912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,8,4,128,1,float16,float16,0,1.0990933577219646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,1,128,1,float16,float16,0,0.5196959972381592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,1,128,1,float16,fp8,0,0.5208693345387777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,8,128,1,float16,float16,0,0.6454399824142456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,8,128,1,float16,fp8,0,0.6189866860707601
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,8,128,1,fp8,fp8,0,0.5589813391367594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,8,4,128,1,float16,fp8,0,1.078768014907837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,8,4,128,1,fp8,fp8,0,1.0380213260650635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,1,128,1,fp8,fp8,0,0.4515039920806885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,2,128,1,float16,float16,0,0.5317279895146688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,2,128,1,float16,fp8,0,0.5284159978230795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,2,128,1,fp8,fp8,0,0.4726879994074504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,4,128,1,float16,fp8,0,0.5514826774597168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,8,128,1,float16,float16,0,0.333957314491272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,4,128,1,fp8,fp8,0,0.5297013521194458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,8,4,128,1,float16,float16,0,0.5637120008468628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,8,128,1,float16,fp8,0,0.3235413432121277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,1,128,1,float16,float16,0,0.2736053268114726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,8,128,1,fp8,fp8,0,0.28940266370773315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,1,128,1,float16,fp8,0,0.2735466758410136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,1,128,1,fp8,fp8,0,0.23906133572260538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,2,128,1,float16,float16,0,0.279039998849233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,2,128,1,float16,fp8,0,0.2783466577529907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,2,128,1,fp8,fp8,0,0.24816532929738364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,4,128,1,float16,float16,0,0.29630933205286664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,4,128,1,float16,fp8,0,0.2924000024795532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,8,128,1,float16,float16,0,0.18182400862375894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,8,4,128,1,fp8,fp8,0,0.2745866576830546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,8,128,1,float16,fp8,0,0.1750346620877584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,8,128,1,fp8,fp8,0,0.15220266580581665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,1,128,1,float16,float16,0,0.15194666385650635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,1,128,1,float16,fp8,0,0.15068266789118448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,1,128,1,fp8,fp8,0,0.12668800354003906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,2,128,1,float16,fp8,0,0.15263467033704123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,2,128,1,fp8,fp8,0,0.1302239994208018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,2,128,1,float16,float16,0,0.15532799561818442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,4,128,1,float16,float16,0,0.16265066464742026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,4,128,1,float16,fp8,0,0.16061333815256754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,8,4,128,1,fp8,fp8,0,0.1404853363831838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,8,128,1,float16,float16,0,0.10040000081062317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,1,128,1,float16,float16,0,0.07994666695594788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,8,128,1,float16,fp8,0,0.09694400429725647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,8,128,1,fp8,fp8,0,0.08450667063395183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,1,128,1,fp8,fp8,0,0.07112533350785573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,1,128,1,float16,fp8,0,0.0800799975792567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,2,128,1,float16,float16,0,0.08251200119654338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,2,128,1,float16,fp8,0,0.08229866623878479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,2,128,1,fp8,fp8,0,0.07330133517583211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,4,128,1,float16,float16,0,0.08642133076985677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,4,128,1,float16,fp8,0,0.0860586663087209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,8,128,1,float16,float16,0,0.05381333331267039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,8,128,1,float16,fp8,0,0.0521919975678126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,8,4,128,1,fp8,fp8,0,0.07735466460386912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,1,128,1,float16,fp8,0,0.046336000164349876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,1,128,1,float16,float16,0,0.04754666487375895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,8,128,1,fp8,fp8,0,0.051029334465662636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,1,128,1,fp8,fp8,0,0.041333332657814026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,2,128,1,float16,float16,0,0.047695999344189964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,2,128,1,float16,fp8,0,0.04690133531888326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,4,128,1,float16,float16,0,0.04960533479849497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,4,128,1,float16,fp8,0,0.04841066896915436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,2,128,1,fp8,fp8,0,0.046442667643229164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,8,128,1,float16,fp8,0,0.03696533292531967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,8,4,128,1,fp8,fp8,0,0.046570668617884316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,8,128,1,float16,float16,0,0.037248000502586365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,8,128,1,fp8,fp8,0,0.0320266659061114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,1,128,1,float16,float16,0,0.035418666899204254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,1,128,1,float16,fp8,0,0.03505066782236099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,2,128,1,float16,float16,0,0.03620799879233042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,4,128,1,float16,float16,0,0.03620799879233042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,4,128,1,float16,fp8,0,0.036288000643253326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,2,128,1,float16,fp8,0,0.03610666592915853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,1,128,1,fp8,fp8,0,0.030933332939942677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,4,128,1,fp8,fp8,0,0.032245332996050514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,8,2,128,1,fp8,fp8,0,0.031311998764673867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,8,128,1,float16,float16,0,0.03334933271010717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,8,128,1,float16,fp8,0,0.032399999598662056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,1,128,1,float16,float16,0,0.032325332363446556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,1,128,1,fp8,fp8,0,0.02824000020821889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,8,128,1,fp8,fp8,0,0.028736000259717304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,1,128,1,float16,fp8,0,0.03244800120592117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,2,128,1,fp8,fp8,0,0.028202667832374573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,2,128,1,float16,fp8,0,0.03203733265399933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,2,128,1,float16,float16,0,0.03257066756486893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,4,128,1,float16,float16,0,0.03254399945338567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,4,128,1,float16,fp8,0,0.03270933280388514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,8,4,128,1,fp8,fp8,0,0.029088000456492107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,8,1,128,1,float16,fp8,0,0.7884480158487955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,8,1,128,1,fp8,fp8,0,0.7285760243733724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,8,1,128,1,float16,float16,0,0.7937173048655192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,8,2,128,1,float16,fp8,0,0.8108586470286051
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,8,2,128,1,fp8,fp8,0,0.7872693538665771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,8,2,128,1,float16,float16,0,0.8171146710713705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,8,4,128,1,float16,float16,0,0.8797120253245035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,1,128,1,float16,float16,0,0.40857064723968506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,1,128,1,float16,fp8,0,0.40651198228200275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,8,128,1,float16,float16,0,0.542794664700826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,8,4,128,1,float16,fp8,0,0.8606506983439127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,8,128,1,fp8,fp8,0,0.48094932238260907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,8,128,1,float16,fp8,0,0.514842669169108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,8,4,128,1,fp8,fp8,0,0.8866026401519775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,1,128,1,fp8,fp8,0,0.3762773275375366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,2,128,1,float16,float16,0,0.42056000232696533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,2,128,1,float16,fp8,0,0.4166026512781779
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,2,128,1,fp8,fp8,0,0.4008479913075765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,4,128,1,float16,float16,0,0.4532959858576457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,4,128,1,float16,fp8,0,0.442415992418925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,8,128,1,float16,float16,0,0.2791200081507365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,8,4,128,1,fp8,fp8,0,0.45135998725891113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,1,128,1,float16,float16,0,0.21589332818984985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,8,128,1,float16,fp8,0,0.2695573369661967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,8,128,1,fp8,fp8,0,0.2504426638285319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,1,128,1,float16,fp8,0,0.21410665909449259
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,1,128,1,fp8,fp8,0,0.1992853283882141
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,2,128,1,float16,float16,0,0.22246400515238443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,2,128,1,float16,fp8,0,0.219760000705719
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,2,128,1,fp8,fp8,0,0.2062879999478658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,4,128,1,float16,float16,0,0.2388319969177246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,4,128,1,float16,fp8,0,0.2339893380800883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,8,128,1,float16,float16,0,0.1524853308995565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,8,128,1,float16,fp8,0,0.14679466684659323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,8,4,128,1,fp8,fp8,0,0.23370667298634848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,1,128,1,float16,float16,0,0.12040533622105916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,8,128,1,fp8,fp8,0,0.1343839963277181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,1,128,1,float16,fp8,0,0.11990933616956075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,1,128,1,fp8,fp8,0,0.11009066303571065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,2,128,1,float16,float16,0,0.12377066413561504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,2,128,1,float16,fp8,0,0.12214932839075725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,2,128,1,fp8,fp8,0,0.11297600467999776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,4,128,1,float16,float16,0,0.13377599914868674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,4,128,1,float16,fp8,0,0.12965866923332214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,8,4,128,1,fp8,fp8,0,0.1211946705977122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,8,128,1,float16,float16,0,0.08781333764394124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,8,128,1,float16,fp8,0,0.08535466591517131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,1,128,1,float16,fp8,0,0.0669653316338857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,1,128,1,float16,float16,0,0.06666133304437001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,8,128,1,fp8,fp8,0,0.07553599774837494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,2,128,1,float16,float16,0,0.06926933427651723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,1,128,1,fp8,fp8,0,0.061679999033610024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,4,128,1,float16,float16,0,0.07357866565386455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,2,128,1,fp8,fp8,0,0.06523199876149495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,4,128,1,float16,fp8,0,0.0721919983625412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,2,128,1,float16,fp8,0,0.06968000034491222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,8,4,128,1,fp8,fp8,0,0.0676746666431427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,8,128,1,float16,float16,0,0.04794666667779287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,1,128,1,float16,float16,0,0.040378667414188385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,1,128,1,float16,fp8,0,0.03994133323431015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,8,128,1,float16,fp8,0,0.04574400186538696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,8,128,1,fp8,fp8,0,0.04595200220743815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,2,128,1,float16,fp8,0,0.04101866732041041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,1,128,1,fp8,fp8,0,0.03621333340803782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,4,128,1,float16,float16,0,0.04279999931653341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,2,128,1,float16,float16,0,0.044682666659355164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,2,128,1,fp8,fp8,0,0.03718933214743932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,4,128,1,float16,fp8,0,0.04164800047874451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,8,4,128,1,fp8,fp8,0,0.0415040006240209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,8,128,1,float16,float16,0,0.032144000132878624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,8,128,1,float16,fp8,0,0.031727999448776245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,8,128,1,fp8,fp8,0,0.028394666810830433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,1,128,1,float16,fp8,0,0.029317334294319153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,1,128,1,fp8,fp8,0,0.026186667382717133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,2,128,1,float16,float16,0,0.029637334247430164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,1,128,1,float16,float16,0,0.029818666477998097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,2,128,1,float16,fp8,0,0.029674666623274486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,2,128,1,fp8,fp8,0,0.027093333502610523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,4,128,1,float16,float16,0,0.03073066721359889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,4,128,1,float16,fp8,0,0.030933332939942677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,8,128,1,float16,float16,0,0.026528000831604004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,8,4,128,1,fp8,fp8,0,0.028079998989899952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,8,128,1,fp8,fp8,0,0.024245334168275196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,1,128,1,float16,fp8,0,0.025861332813898723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,1,128,1,fp8,fp8,0,0.023823998868465424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,2,128,1,float16,float16,0,0.02532266577084859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,8,128,1,float16,fp8,0,0.027248000105222065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,2,128,1,float16,fp8,0,0.025663999219735462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,1,128,1,float16,float16,0,0.025455998877684276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,2,128,1,fp8,fp8,0,0.02405333270629247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,4,128,1,fp8,fp8,0,0.024234667420387268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,4,128,1,float16,float16,0,0.025957333544890087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,8,4,128,1,float16,fp8,0,0.02622399975856145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,8,128,1,float16,float16,0,0.02430933217207591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,8,128,1,float16,fp8,0,0.024512000381946564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,1,128,1,float16,fp8,0,0.0244159996509552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,1,128,1,float16,float16,0,0.0245919997493426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,1,128,1,fp8,fp8,0,0.02254933367172877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,2,128,1,float16,float16,0,0.02458133300145467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,8,128,1,fp8,fp8,0,0.022250667214393616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,2,128,1,float16,fp8,0,0.02516266703605652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,4,128,1,float16,float16,0,0.024533333877722423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,2,128,1,fp8,fp8,0,0.023290666441122692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,4,128,1,float16,fp8,0,0.024911999702453613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,8,4,128,1,fp8,fp8,0,0.02271999915440877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,8,1,128,1,float16,float16,0,0.3497493267059326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,8,1,128,1,fp8,fp8,0,0.32833067576090497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,8,1,128,1,float16,fp8,0,0.34849599997202557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,8,2,128,1,float16,fp8,0,0.3593226671218872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,8,2,128,1,float16,float16,0,0.36101333300272626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,8,2,128,1,fp8,fp8,0,0.3562026818593343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,8,4,128,1,float16,float16,0,0.3978079954783122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,1,128,1,float16,float16,0,0.18473599354426065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,8,128,1,float16,float16,0,0.2571199933687846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,1,128,1,float16,fp8,0,0.18430399894714355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,8,128,1,fp8,fp8,0,0.2273706595102946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,8,4,128,1,float16,fp8,0,0.38874133427937824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,8,4,128,1,fp8,fp8,0,0.40324799219767254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,8,128,1,float16,fp8,0,0.24555200338363647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,1,128,1,fp8,fp8,0,0.17708800236384073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,2,128,1,fp8,fp8,0,0.18494399388631186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,2,128,1,float16,fp8,0,0.1902559995651245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,2,128,1,float16,float16,0,0.19142399231592813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,4,128,1,float16,fp8,0,0.2046133279800415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,4,128,1,float16,float16,0,0.2111733357111613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,8,4,128,1,fp8,fp8,0,0.20974934101104736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,8,128,1,float16,float16,0,0.14114133516947427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,1,128,1,float16,float16,0,0.10439999898274739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,8,128,1,float16,fp8,0,0.1365066667397817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,1,128,1,float16,fp8,0,0.10449600219726562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,8,128,1,fp8,fp8,0,0.12361600001653035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,1,128,1,fp8,fp8,0,0.10050132870674133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,2,128,1,float16,fp8,0,0.10630399982134502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,2,128,1,float16,float16,0,0.10875733693440755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,2,128,1,fp8,fp8,0,0.1042133371035258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,4,128,1,float16,float16,0,0.11877333124478658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,4,128,1,float16,fp8,0,0.11575999855995178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,8,128,1,float16,float16,0,0.08402132987976074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,8,128,1,float16,fp8,0,0.08140799899895985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,8,4,128,1,fp8,fp8,0,0.11246933539708455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,8,128,1,fp8,fp8,0,0.0703413337469101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,1,128,1,float16,fp8,0,0.060378665725390114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,1,128,1,float16,float16,0,0.06090133388837179
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,2,128,1,float16,fp8,0,0.06241066753864288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,2,128,1,float16,float16,0,0.06198933223883311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,1,128,1,fp8,fp8,0,0.05682666599750519
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,2,128,1,fp8,fp8,0,0.05901333192984263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,4,128,1,float16,float16,0,0.06670400003592174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,4,128,1,float16,fp8,0,0.06624533236026764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,8,128,1,float16,float16,0,0.04447466631730398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,1,128,1,float16,float16,0,0.037818667789300285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,8,4,128,1,fp8,fp8,0,0.06413333117961884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,8,128,1,float16,fp8,0,0.043098668257395424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,1,128,1,float16,fp8,0,0.03757333258787791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,8,128,1,fp8,fp8,0,0.04192533095677694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,1,128,1,fp8,fp8,0,0.033402666449546814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,2,128,1,float16,float16,0,0.038106667498747505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,2,128,1,float16,fp8,0,0.03796799977620443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,2,128,1,fp8,fp8,0,0.03465066601832708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,4,128,1,float16,float16,0,0.04125333329041799
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,4,128,1,float16,fp8,0,0.03982933362325033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,8,4,128,1,fp8,fp8,0,0.037946666280428566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,8,128,1,float16,float16,0,0.028255999088287354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,8,128,1,fp8,fp8,0,0.025066666305065155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,1,128,1,float16,fp8,0,0.026170666019121807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,1,128,1,fp8,fp8,0,0.023498666783173878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,2,128,1,float16,fp8,0,0.026176000634829204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,2,128,1,float16,float16,0,0.026693334182103474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,1,128,1,float16,float16,0,0.025600001215934753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,8,128,1,float16,fp8,0,0.02842666705449422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,4,128,1,fp8,fp8,0,0.02497600018978119
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,2,128,1,fp8,fp8,0,0.023541333774725597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,4,128,1,float16,float16,0,0.027989332874615986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,8,4,128,1,float16,fp8,0,0.02752000093460083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,8,128,1,float16,float16,0,0.02327999969323476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,1,128,1,float16,float16,0,0.022181332111358643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,8,128,1,fp8,fp8,0,0.021690666675567627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,8,128,1,float16,fp8,0,0.023205332458019257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,1,128,1,float16,fp8,0,0.02203733225663503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,1,128,1,fp8,fp8,0,0.021210665504137676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,2,128,1,float16,float16,0,0.02274666726589203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,2,128,1,float16,fp8,0,0.022064000368118286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,2,128,1,fp8,fp8,0,0.020773333807786305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,4,128,1,float16,float16,0,0.02260799954334895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,1,128,1,float16,float16,0,0.02077866718173027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,4,128,1,fp8,fp8,0,0.021509334444999695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,8,128,1,float16,float16,0,0.02141333371400833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,8,128,1,float16,fp8,0,0.021061333517233532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,8,128,1,fp8,fp8,0,0.01982933282852173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,8,4,128,1,float16,fp8,0,0.022778667509555817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,1,128,1,fp8,fp8,0,0.019253333409627277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,1,128,1,float16,fp8,0,0.021456000705560047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,2,128,1,float16,float16,0,0.02096533278624217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,2,128,1,float16,fp8,0,0.021375998854637146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,4,128,1,float16,float16,0,0.021173333128293354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,2,128,1,fp8,fp8,0,0.01982933282852173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,4,128,1,fp8,fp8,0,0.0194560003777345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,8,128,1,float16,float16,0,0.01964266722400983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,8,128,1,float16,fp8,0,0.020848001043001812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,8,4,128,1,float16,fp8,0,0.021781332790851593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,1,128,1,fp8,fp8,0,0.01886933296918869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,1,128,1,float16,fp8,0,0.02035733312368393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,1,128,1,float16,float16,0,0.020549333343903225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,2,128,1,float16,float16,0,0.02015999952952067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,8,128,1,fp8,fp8,0,0.019023999571800232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,2,128,1,float16,fp8,0,0.021066665649414062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,4,128,1,float16,float16,0,0.020597333709398907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,4,128,1,float16,fp8,0,0.020810666183630627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,4,128,1,fp8,fp8,0,0.019461333751678467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,8,2,128,1,fp8,fp8,0,0.01931200052301089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,8,1,128,1,float16,float16,0,0.17782400051752725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,8,1,128,1,fp8,fp8,0,0.17510400215784708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,8,1,128,1,float16,fp8,0,0.1756640076637268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,8,2,128,1,float16,float16,0,0.18286399046579996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,8,2,128,1,float16,fp8,0,0.18193066120147705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,8,2,128,1,fp8,fp8,0,0.1836586594581604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,8,4,128,1,float16,float16,0,0.20189867417017618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,1,128,1,float16,float16,0,0.10195733110109965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,8,4,128,1,float16,fp8,0,0.19723733266194662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,8,4,128,1,fp8,fp8,0,0.2091360092163086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,8,128,1,fp8,fp8,0,0.12035199999809265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,8,128,1,float16,fp8,0,0.1399946709473928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,8,128,1,float16,float16,0,0.1442453364531199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,1,128,1,float16,fp8,0,0.1011253297328949
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,2,128,1,float16,float16,0,0.10453333457310994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,1,128,1,fp8,fp8,0,0.09967999656995137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,2,128,1,float16,fp8,0,0.10389332969983418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,2,128,1,fp8,fp8,0,0.10231467088063557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,4,128,1,float16,float16,0,0.11542399724324544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,4,128,1,float16,fp8,0,0.11387200156847636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,1,128,1,float16,float16,0,0.05745600163936615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,8,128,1,float16,float16,0,0.0794293334086736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,1,128,1,float16,fp8,0,0.05771199862162272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,8,128,1,float16,fp8,0,0.0757013310988744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,8,128,1,fp8,fp8,0,0.06948266426722209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,8,4,128,1,fp8,fp8,0,0.11283199985822041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,1,128,1,fp8,fp8,0,0.05765866736570994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,2,128,1,fp8,fp8,0,0.059152002135912575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,2,128,1,float16,float16,0,0.05977599819501241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,4,128,1,float16,fp8,0,0.06435200075308482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,4,128,1,float16,float16,0,0.06761066615581512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,8,128,1,float16,float16,0,0.0430026650428772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,2,128,1,float16,fp8,0,0.05930666625499725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,8,4,128,1,fp8,fp8,0,0.06355733176072438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,8,128,1,float16,fp8,0,0.04112533231576284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,1,128,1,float16,fp8,0,0.03598399957021078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,8,128,1,fp8,fp8,0,0.0415040006240209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,1,128,1,float16,float16,0,0.03612266729275385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,2,128,1,float16,float16,0,0.03714133302370707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,2,128,1,fp8,fp8,0,0.03366933266321818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,1,128,1,fp8,fp8,0,0.034341332813103996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,4,128,1,float16,float16,0,0.03847466657559077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,2,128,1,float16,fp8,0,0.036677333215872444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,4,128,1,float16,fp8,0,0.03801066676775614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,8,4,128,1,fp8,fp8,0,0.03766933331886927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,8,128,1,float16,float16,0,0.027647999425729115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,8,128,1,float16,fp8,0,0.026746665438016255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,8,128,1,fp8,fp8,0,0.025472000241279602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,1,128,1,float16,float16,0,0.025888000925381977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,1,128,1,fp8,fp8,0,0.023258666197458904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,2,128,1,float16,float16,0,0.025786665578683216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,2,128,1,float16,fp8,0,0.02605333427588145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,1,128,1,float16,fp8,0,0.025397333006064098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,2,128,1,fp8,fp8,0,0.02380799998839696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,8,128,1,float16,fp8,0,0.021354667842388153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,4,128,1,float16,fp8,0,0.026917333404223125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,4,128,1,float16,float16,0,0.02681066592534383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,8,4,128,1,fp8,fp8,0,0.024986666937669117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,8,128,1,float16,float16,0,0.02181866765022278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,8,128,1,fp8,fp8,0,0.02139200021823247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,1,128,1,float16,fp8,0,0.02080533280968666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,1,128,1,float16,float16,0,0.02042666698495547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,1,128,1,fp8,fp8,0,0.020981334149837494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,2,128,1,fp8,fp8,0,0.020794666061798733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,2,128,1,float16,fp8,0,0.020949333906173706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,2,128,1,float16,float16,0,0.020703999946514767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,4,128,1,float16,float16,0,0.02149333308140437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,4,128,1,float16,fp8,0,0.02144533395767212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,8,4,128,1,fp8,fp8,0,0.021344001094500225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,8,128,1,float16,float16,0,0.01969066634774208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,8,128,1,float16,fp8,0,0.019808000574509304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,8,128,1,fp8,fp8,0,0.01961600035429001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,1,128,1,float16,float16,0,0.019493332753578823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,2,128,1,float16,float16,0,0.019493332753578823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,1,128,1,float16,fp8,0,0.01993600030740102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,2,128,1,fp8,fp8,0,0.019578666736682255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,2,128,1,float16,fp8,0,0.0200853335360686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,1,128,1,fp8,fp8,0,0.01942933350801468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,4,128,1,float16,float16,0,0.01959466685851415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,4,128,1,fp8,fp8,0,0.019765333582957584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,8,4,128,1,float16,fp8,0,0.020096000283956528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,8,128,1,float16,fp8,0,0.019205333044131596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,1,128,1,fp8,fp8,0,0.019306667149066925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,8,128,1,fp8,fp8,0,0.018618666877349217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,8,128,1,float16,float16,0,0.019007999449968338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,1,128,1,float16,fp8,0,0.019317333896954853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,2,128,1,fp8,fp8,0,0.019146667172511418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,1,128,1,float16,float16,0,0.019152000546455383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,2,128,1,float16,fp8,0,0.01913600042462349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,2,128,1,float16,float16,0,0.019248000035683315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,4,128,1,float16,fp8,0,0.019685332973798115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,4,128,1,float16,float16,0,0.018725333114465077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,8,4,128,1,fp8,fp8,0,0.019413333386182785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,8,128,1,float16,float16,0,0.018245333184798557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,8,128,1,fp8,fp8,0,0.0186666672428449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,1,128,1,float16,fp8,0,0.01893866683046023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,1,128,1,float16,float16,0,0.018917333334684372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,1,128,1,fp8,fp8,0,0.018944000204404194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,2,128,1,float16,fp8,0,0.018650667121013004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,2,128,1,float16,float16,0,0.018320000420014065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,8,128,1,float16,fp8,0,0.01874133323629697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,4,128,1,float16,float16,0,0.01830400029818217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,2,128,1,fp8,fp8,0,0.018751999984184902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,4,128,1,fp8,fp8,0,0.018709332992633183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,8,4,128,1,float16,fp8,0,0.019440000255902607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,8,1,128,1,float16,float16,0,0.11537599563598633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,8,1,128,1,float16,fp8,0,0.11476266384124756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,8,1,128,1,fp8,fp8,0,0.12558933099110922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,8,2,128,1,float16,float16,0,0.11915733416875203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,8,2,128,1,float16,fp8,0,0.11865599950154622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,8,2,128,1,fp8,fp8,0,0.12956800063451132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,8,4,128,1,float16,float16,0,0.1325546701749166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,8,4,128,1,float16,fp8,0,0.12867200374603271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,8,4,128,1,fp8,fp8,0,0.13828800121943155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,8,128,1,float16,float16,0,0.07899199922879536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,1,128,1,float16,float16,0,0.06471999982992808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,8,128,1,fp8,fp8,0,0.08226133386294048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,8,128,1,float16,fp8,0,0.0759200006723404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,1,128,1,fp8,fp8,0,0.07116800049940745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,2,128,1,float16,float16,0,0.06614933411280315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,1,128,1,float16,fp8,0,0.06505066653092702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,2,128,1,float16,fp8,0,0.06542933483918507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,2,128,1,fp8,fp8,0,0.07281066477298737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,4,128,1,float16,float16,0,0.07232533395290375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,4,128,1,fp8,fp8,0,0.07776000102361043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,8,128,1,float16,float16,0,0.043706665436426796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,8,128,1,fp8,fp8,0,0.048672000567118325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,1,128,1,float16,float16,0,0.0391893337170283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,8,128,1,float16,fp8,0,0.042026668787002563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,1,128,1,float16,fp8,0,0.03851733356714249
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,8,4,128,1,float16,fp8,0,0.07107733190059662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,1,128,1,fp8,fp8,0,0.04091733445723852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,2,128,1,float16,float16,0,0.03938666731119156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,2,128,1,fp8,fp8,0,0.04229333500067393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,2,128,1,float16,fp8,0,0.03923733284076055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,4,128,1,float16,fp8,0,0.04031466692686081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,4,128,1,float16,float16,0,0.04156800111134847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,8,4,128,1,fp8,fp8,0,0.04488533238569895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,8,128,1,float16,float16,0,0.028373333315054577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,8,128,1,float16,fp8,0,0.028346667687098186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,1,128,1,float16,float16,0,0.026858667532602947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,8,128,1,fp8,fp8,0,0.02887466549873352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,1,128,1,float16,fp8,0,0.02672533442576726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,1,128,1,fp8,fp8,0,0.027999999622503918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,2,128,1,float16,fp8,0,0.027189334233601887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,2,128,1,fp8,fp8,0,0.027834666272004444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,2,128,1,float16,float16,0,0.027317332724730175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,4,128,1,float16,float16,0,0.02802666773398717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,4,128,1,float16,fp8,0,0.028021333118279774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,8,4,128,1,fp8,fp8,0,0.028565332293510437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,8,128,1,float16,float16,0,0.02147199958562851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,8,128,1,float16,fp8,0,0.02096533278624217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,1,128,1,float16,fp8,0,0.02051199972629547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,8,128,1,fp8,fp8,0,0.02207999924818675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,1,128,1,fp8,fp8,0,0.020629333953062694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,1,128,1,float16,float16,0,0.02060266708334287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,2,128,1,float16,float16,0,0.02037866661945979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,2,128,1,fp8,fp8,0,0.020762667059898376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,2,128,1,float16,fp8,0,0.020799999435742695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,4,128,1,float16,float16,0,0.02085866779088974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,4,128,1,float16,fp8,0,0.02083733429511388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,8,4,128,1,fp8,fp8,0,0.02163733293612798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,8,128,1,float16,fp8,0,0.018858666221300762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,8,128,1,fp8,fp8,0,0.01937599976857503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,8,128,1,float16,float16,0,0.01850133389234543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,1,128,1,float16,fp8,0,0.01874133323629697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,1,128,1,fp8,fp8,0,0.019285333653291065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,1,128,1,float16,float16,0,0.01829333355029424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,2,128,1,float16,float16,0,0.018698666244745255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,2,128,1,float16,fp8,0,0.018826667219400406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,2,128,1,fp8,fp8,0,0.019434666881958645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,4,128,1,float16,fp8,0,0.01907733331123988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,4,128,1,fp8,fp8,0,0.019413333386182785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,8,4,128,1,float16,float16,0,0.018506667266289394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,8,128,1,float16,fp8,0,0.018245333184798557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,8,128,1,fp8,fp8,0,0.019141333798567455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,8,128,1,float16,float16,0,0.01809599995613098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,1,128,1,float16,fp8,0,0.018357332795858383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,1,128,1,float16,float16,0,0.017946666727463405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,1,128,1,fp8,fp8,0,0.01903466631968816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,2,128,1,float16,fp8,0,0.01854933301607768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,2,128,1,fp8,fp8,0,0.019002666076024372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,4,128,1,float16,float16,0,0.018277333428462345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,4,128,1,float16,fp8,0,0.018277333428462345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,2,128,1,float16,float16,0,0.01782400036851565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,8,4,128,1,fp8,fp8,0,0.019130667050679524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,8,128,1,float16,float16,0,0.017184000462293625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,8,128,1,float16,fp8,0,0.017871999492247898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,1,128,1,float16,fp8,0,0.0176959993938605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,8,128,1,fp8,fp8,0,0.01834133391578992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,2,128,1,float16,float16,0,0.017504000415404636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,1,128,1,float16,float16,0,0.01777600000301997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,1,128,1,fp8,fp8,0,0.01826133330663045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,2,128,1,float16,fp8,0,0.01788266624013583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,2,128,1,fp8,fp8,0,0.018960000326236088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,4,128,1,float16,float16,0,0.017583999782800674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,4,128,1,float16,fp8,0,0.01758933315674464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,8,4,128,1,fp8,fp8,0,0.01883200059334437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,8,128,1,float16,float16,0,0.017173333714405697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,1,128,1,float16,fp8,0,0.017557332913080852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,8,128,1,fp8,fp8,0,0.01829333355029424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,1,128,1,float16,float16,0,0.01692266638080279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,2,128,1,float16,float16,0,0.01725333308180173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,8,128,1,float16,fp8,0,0.017498667041460674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,1,128,1,fp8,fp8,0,0.01854933301607768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,2,128,1,float16,fp8,0,0.01758933315674464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,2,128,1,fp8,fp8,0,0.018672000616788864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,4,128,1,float16,float16,0,0.017242666333913803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,4,128,1,float16,fp8,0,0.017258666455745697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,8,4,128,1,fp8,fp8,0,0.05789866546789805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,8,1,128,1,float16,fp8,0,0.07947733501593272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,8,1,128,1,fp8,fp8,0,0.10249066352844238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,8,2,128,1,float16,float16,0,0.08105599880218506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,8,1,128,1,float16,float16,0,0.07960533102353413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,8,2,128,1,float16,fp8,0,0.08122133215268452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,8,2,128,1,fp8,fp8,0,0.10505599776903789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,8,4,128,1,float16,float16,0,0.08835732936859131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,8,4,128,1,float16,fp8,0,0.08724266290664673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,8,4,128,1,fp8,fp8,0,0.1090666651725769
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,8,128,1,float16,float16,0,0.050383999943733215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,1,128,1,float16,float16,0,0.04572266836961111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,1,128,1,float16,fp8,0,0.04555733501911163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,8,128,1,float16,fp8,0,0.0497920016447703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,8,128,1,fp8,fp8,0,0.06502399841944377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,2,128,1,float16,fp8,0,0.046015997727712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,1,128,1,fp8,fp8,0,0.05629866818586985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,2,128,1,fp8,fp8,0,0.058287998040517174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,4,128,1,float16,float16,0,0.048384000857671104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,2,128,1,float16,float16,0,0.04640533526738485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,4,128,1,fp8,fp8,0,0.060959999759991966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,8,4,128,1,float16,fp8,0,0.04762133459250132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,8,128,1,float16,float16,0,0.030853333572546642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,1,128,1,float16,float16,0,0.029845332105954487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,8,128,1,float16,fp8,0,0.030581332743167877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,8,128,1,fp8,fp8,0,0.03697066754102707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,1,128,1,float16,fp8,0,0.030165334542592365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,1,128,1,fp8,fp8,0,0.0354720006386439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,2,128,1,fp8,fp8,0,0.03610666592915853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,2,128,1,float16,fp8,0,0.03068800022204717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,4,128,1,float16,float16,0,0.03154666721820831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,2,128,1,float16,float16,0,0.030565333863099415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,4,128,1,float16,fp8,0,0.03151999910672506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,8,4,128,1,fp8,fp8,0,0.03786666691303253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,8,128,1,float16,float16,0,0.022661333282788593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,8,128,1,fp8,fp8,0,0.02534399926662445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,1,128,1,float16,float16,0,0.0220320001244545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,8,128,1,float16,fp8,0,0.022255999346574146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,1,128,1,fp8,fp8,0,0.025061334172884624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,2,128,1,float16,float16,0,0.022858666876951855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,1,128,1,float16,fp8,0,0.02242133269707362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,2,128,1,float16,fp8,0,0.022277332842350006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,2,128,1,fp8,fp8,0,0.025413334369659424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,4,128,1,float16,float16,0,0.02232533444960912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,4,128,1,float16,fp8,0,0.02260799954334895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,8,4,128,1,fp8,fp8,0,0.025829332570234936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,8,128,1,float16,float16,0,0.018415999909241993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,1,128,1,float16,float16,0,0.017871999492247898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,8,128,1,fp8,fp8,0,0.020138667275508244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,8,128,1,float16,fp8,0,0.021221332252025604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,1,128,1,float16,fp8,0,0.01781333362062772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,1,128,1,fp8,fp8,0,0.019434666881958645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,2,128,1,float16,float16,0,0.01788266624013583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,2,128,1,float16,fp8,0,0.018031999468803406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,2,128,1,fp8,fp8,0,0.020069333414236706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,4,128,1,float16,float16,0,0.01801066721479098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,4,128,1,fp8,fp8,0,0.0198186660806338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,8,128,1,float16,float16,0,0.016805333395799
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,8,128,1,float16,fp8,0,0.01732800031701724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,8,4,128,1,float16,fp8,0,0.018042666216691334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,1,128,1,float16,float16,0,0.01764800027012825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,8,128,1,fp8,fp8,0,0.01865600049495697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,1,128,1,float16,fp8,0,0.017610666652520496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,1,128,1,fp8,fp8,0,0.018810667097568512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,2,128,1,float16,float16,0,0.01695999999841054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,4,128,1,float16,float16,0,0.01748266691962878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,2,128,1,float16,fp8,0,0.017583999782800674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,4,128,1,float16,fp8,0,0.017818666994571686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,2,128,1,fp8,fp8,0,0.01859733338157336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,8,128,1,float16,fp8,0,0.01681600014368693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,8,128,1,float16,float16,0,0.016800000021855038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,8,4,128,1,fp8,fp8,0,0.019354666272799175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,8,128,1,fp8,fp8,0,0.01858666663368543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,1,128,1,float16,float16,0,0.017071999609470367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,1,128,1,float16,fp8,0,0.017370666066805523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,2,128,1,float16,float16,0,0.016682667036851246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,2,128,1,float16,fp8,0,0.017375999440749485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,4,128,1,float16,float16,0,0.016832000265518825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,2,128,1,fp8,fp8,0,0.018645333747069042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,1,128,1,fp8,fp8,0,0.028378665447235107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,4,128,1,float16,fp8,0,0.01759999990463257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,8,4,128,1,fp8,fp8,0,0.018954666952292126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,8,128,1,float16,float16,0,0.01609066625436147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,8,128,1,fp8,fp8,0,0.018191999445358913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,8,128,1,float16,fp8,0,0.016869333883126576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,1,128,1,float16,float16,0,0.016399999459584553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,1,128,1,fp8,fp8,0,0.018730666488409042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,2,128,1,float16,float16,0,0.016917333006858826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,1,128,1,float16,fp8,0,0.017157333592573803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,2,128,1,fp8,fp8,0,0.018453333526849747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,2,128,1,float16,fp8,0,0.017071999609470367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,4,128,1,float16,float16,0,0.01704000060757001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,4,128,1,float16,fp8,0,0.016858667135238647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,8,4,128,1,fp8,fp8,0,0.01887999971707662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,8,128,1,fp8,fp8,0,0.018346666047970455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,8,128,1,float16,float16,0,0.0164533331990242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,1,128,1,float16,float16,0,0.01632533346613248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,8,128,1,float16,fp8,0,0.016714667280515034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,1,128,1,float16,fp8,0,0.016821333517630894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,1,128,1,fp8,fp8,0,0.018266666680574417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,2,128,1,float16,float16,0,0.016895999511082966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,2,128,1,float16,fp8,0,0.016901332885026932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,2,128,1,fp8,fp8,0,0.018735999862353008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,4,128,1,float16,float16,0,0.01643199970324834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,4,128,1,fp8,fp8,0,0.01800000046690305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,8,1,128,1,fp8,fp8,0,0.08914666374524434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,8,1,128,1,float16,fp8,0,0.06816533207893372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,8,4,128,1,float16,fp8,0,0.025578667720158894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,8,1,128,1,float16,float16,0,0.06782400111357371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,8,2,128,1,float16,float16,0,0.06861333549022675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,8,2,128,1,float16,fp8,0,0.06821866830190022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,8,2,128,1,fp8,fp8,0,0.08941333492596944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,8,4,128,1,float16,float16,0,0.07074133555094402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,8,4,128,1,float16,fp8,0,0.07010133564472198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,8,128,1,float16,float16,0,0.040149333576361336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,8,128,1,fp8,fp8,0,0.0529120018084844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,8,4,128,1,fp8,fp8,0,0.09397332866986592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,8,128,1,float16,fp8,0,0.03984000037113825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,1,128,1,float16,float16,0,0.04084266722202301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,1,128,1,float16,fp8,0,0.04144000013669332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,1,128,1,fp8,fp8,0,0.05157866577307383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,2,128,1,float16,fp8,0,0.040991999208927155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,2,128,1,float16,float16,0,0.041663999358812966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,4,128,1,float16,float16,0,0.041850666205088295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,4,128,1,float16,fp8,0,0.042090664307276406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,2,128,1,fp8,fp8,0,0.05235733091831207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,8,4,128,1,fp8,fp8,0,0.0537013312180837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,8,128,1,float16,fp8,0,0.02716800073782603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,8,128,1,fp8,fp8,0,0.033215999603271484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,8,128,1,float16,float16,0,0.02700799951950709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,1,128,1,float16,float16,0,0.02775466690460841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,1,128,1,float16,fp8,0,0.02771199991305669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,2,128,1,float16,float16,0,0.02789866675933202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,2,128,1,float16,fp8,0,0.028229333460330963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,1,128,1,fp8,fp8,0,0.03341866781314214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,2,128,1,fp8,fp8,0,0.03383466601371765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,4,128,1,float16,float16,0,0.028250666956106823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,4,128,1,float16,fp8,0,0.028143999477227528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,8,4,128,1,fp8,fp8,0,0.03383466601371765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,8,128,1,float16,float16,0,0.020432000358899433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,8,128,1,float16,fp8,0,0.020645332833131153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,1,128,1,float16,float16,0,0.020848001043001812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,1,128,1,fp8,fp8,0,0.024192000428835552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,1,128,1,float16,fp8,0,0.02146666745344798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,2,128,1,float16,fp8,0,0.021503999829292297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,8,128,1,fp8,fp8,0,0.02384000023206075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,2,128,1,float16,float16,0,0.020479999482631683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,2,128,1,fp8,fp8,0,0.024277334411938984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,4,128,1,float16,float16,0,0.020597333709398907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,8,128,1,float16,float16,0,0.017797333498795826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,4,128,1,fp8,fp8,0,0.024336000283559162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,8,4,128,1,float16,fp8,0,0.02128000060717265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,8,128,1,fp8,fp8,0,0.019946667055288952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,8,128,1,float16,fp8,0,0.017738666385412216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,1,128,1,float16,float16,0,0.01735466718673706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,1,128,1,fp8,fp8,0,0.019226666539907455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,1,128,1,float16,fp8,0,0.01736533393462499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,2,128,1,float16,float16,0,0.017583999782800674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,2,128,1,float16,fp8,0,0.01782400036851565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,2,128,1,fp8,fp8,0,0.01915733392039935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,4,128,1,fp8,fp8,0,0.019498666127522785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,8,128,1,float16,float16,0,0.016938666502634685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,4,128,1,float16,fp8,0,0.01807466646035512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,8,4,128,1,float16,float16,0,0.017344000438849132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,8,128,1,float16,fp8,0,0.01756799966096878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,8,128,1,fp8,fp8,0,0.018976000448067982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,1,128,1,float16,float16,0,0.016688000410795212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,1,128,1,float16,fp8,0,0.01748266691962878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,1,128,1,fp8,fp8,0,0.01850133389234543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,2,128,1,float16,float16,0,0.01681600014368693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,4,128,1,fp8,fp8,0,0.01859733338157336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,2,128,1,fp8,fp8,0,0.018917333334684372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,4,128,1,float16,float16,0,0.01725333308180173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,4,128,1,float16,fp8,0,0.01708799973130226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,8,2,128,1,float16,fp8,0,0.01729600007335345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,8,128,1,float16,float16,0,0.01669866715868314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,8,128,1,fp8,fp8,0,0.01850133389234543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,8,128,1,float16,fp8,0,0.016549333930015564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,1,128,1,float16,float16,0,0.016837333639462788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,1,128,1,float16,fp8,0,0.017269333203633625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,1,128,1,fp8,fp8,0,0.0184906671444575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,2,128,1,float16,float16,0,0.01701333373785019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,2,128,1,fp8,fp8,0,0.01855466639002164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,2,128,1,float16,fp8,0,0.01741333305835724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,4,128,1,float16,float16,0,0.016421332955360413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,4,128,1,float16,fp8,0,0.016965333372354507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,8,4,128,1,fp8,fp8,0,0.018101333330074947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,8,128,1,float16,float16,0,0.016095999628305435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,8,128,1,fp8,fp8,0,0.01820266619324684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,8,128,1,float16,fp8,0,0.016885332763195038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,1,128,1,float16,float16,0,0.016645333419243496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,1,128,1,float16,fp8,0,0.01682666689157486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,1,128,1,fp8,fp8,0,0.01846933364868164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,2,128,1,float16,float16,0,0.016751999656359356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,2,128,1,float16,fp8,0,0.017221332838137943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,2,128,1,fp8,fp8,0,0.018373332917690277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,4,128,1,fp8,fp8,0,0.018506667266289394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,4,128,1,float16,fp8,0,0.01721599946419398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,1,128,1,float16,float16,0,0.016362667083740234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,8,4,128,1,float16,float16,0,0.016901332885026932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,8,128,1,fp8,fp8,0,0.017850667238235474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,8,128,1,float16,float16,0,0.01597333326935768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,8,128,1,float16,fp8,0,0.016384000579516094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,1,128,1,float16,fp8,0,0.016672000288963318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,1,128,1,fp8,fp8,0,0.01811733345190684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,2,128,1,float16,float16,0,0.01632533346613248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,2,128,1,fp8,fp8,0,0.01836799954374631
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,4,128,1,float16,float16,0,0.016656000167131424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,2,128,1,float16,fp8,0,0.01710933322707812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,4,128,1,float16,fp8,0,0.01725333308180173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,8,4,128,1,fp8,fp8,0,0.018330667167901993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,4,1,128,1,float16,float16,0,4.087589263916016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,4,1,128,1,float16,fp8,0,4.099130630493164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,4,1,128,1,fp8,fp8,0,2.711941401163737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,4,4,128,1,float16,float16,0,2.0415627161661782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,4,2,128,1,fp8,fp8,0,2.729829470316569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,4,2,128,1,float16,float16,0,3.993194580078125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16384,4,2,128,1,float16,fp8,0,4.228101412455241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,4,1,128,1,float16,float16,0,2.0760587056477866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,4,1,128,1,float16,fp8,0,2.0588320096333823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,4,4,128,1,float16,fp8,0,2.0820372899373374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,4,4,128,1,fp8,fp8,0,1.4228800137837727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,4,1,128,1,fp8,fp8,0,1.425279935201009
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,4,2,128,1,float16,float16,0,2.077829360961914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,4,2,128,1,float16,fp8,0,2.0816426277160645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,4,4,128,1,fp8,fp8,0,0.6978027025858561
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,4,4,128,1,float16,float16,0,1.1025866667429607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,4,4,128,1,float16,fp8,0,1.1184639930725098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,4,1,128,1,float16,float16,0,1.1256213188171387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16384,4,2,128,1,fp8,fp8,0,1.4307039578755696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,4,1,128,1,float16,fp8,0,1.1155253251393635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,4,1,128,1,fp8,fp8,0,0.6957386334737142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,4,2,128,1,float16,fp8,0,1.1190880139668782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,4,4,128,1,float16,float16,0,0.5156426827112833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,4,2,128,1,float16,float16,0,1.1284586588541667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,4,4,128,1,fp8,fp8,0,0.3701333204905192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16384,4,2,128,1,fp8,fp8,0,0.7019573052724203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,4,4,128,1,float16,fp8,0,0.5092053413391113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,4,1,128,1,float16,float16,0,0.5194293260574341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,4,1,128,1,float16,fp8,0,0.5143893162409464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,4,2,128,1,float16,float16,0,0.5220213333765665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,4,2,128,1,float16,fp8,0,0.517136017481486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,4,2,128,1,fp8,fp8,0,0.37300264835357666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16384,4,1,128,1,fp8,fp8,0,0.3710506757100423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,4,1,128,1,float16,float16,0,2.354976018269857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,4,1,128,1,fp8,fp8,0,1.6334826151529949
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,4,1,128,1,float16,fp8,0,2.3447093963623047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,4,4,128,1,float16,float16,0,1.2343253294626872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,4,2,128,1,float16,float16,0,2.3766719500223794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,4,2,128,1,fp8,fp8,0,1.6477012634277344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,12288,4,2,128,1,float16,fp8,0,2.3426186243693032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,4,4,128,1,float16,fp8,0,1.2238667011260986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,4,4,128,1,fp8,fp8,0,0.8168533643086752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,4,1,128,1,float16,fp8,0,1.2366399765014648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,4,1,128,1,fp8,fp8,0,0.8097013632456461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,4,1,128,1,float16,float16,0,1.247765302658081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,4,2,128,1,float16,float16,0,1.2687466939290364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,4,2,128,1,float16,fp8,0,1.2386293411254883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,4,4,128,1,float16,float16,0,0.5856159925460815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,4,4,128,1,fp8,fp8,0,0.4286880095799764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,12288,4,2,128,1,fp8,fp8,0,0.8192213376363119
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,4,1,128,1,float16,float16,0,0.5881706476211548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,4,1,128,1,float16,fp8,0,0.5878080129623413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,4,1,128,1,fp8,fp8,0,0.4240479866663615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,4,4,128,1,float16,fp8,0,0.5867466529210409
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,4,2,128,1,float16,fp8,0,0.5877120097478231
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,4,1,128,1,float16,float16,0,0.336736003557841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,4,2,128,1,fp8,fp8,0,0.4278080066045125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,4,4,128,1,float16,fp8,0,0.3345973491668701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,4,4,128,1,float16,float16,0,0.3351840178171794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,4,4,128,1,fp8,fp8,0,0.2466986576716105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,12288,4,2,128,1,float16,float16,0,0.5913973251978556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,4,1,128,1,float16,fp8,0,0.33454398314158124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,4,1,128,1,fp8,fp8,0,0.24440000454584757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,4,2,128,1,float16,float16,0,0.3395306666692098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,4,2,128,1,float16,fp8,0,0.33683733145395917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,12288,4,2,128,1,fp8,fp8,0,0.24729599555333456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,4,1,128,1,float16,float16,0,1.6937813758850098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,4,1,128,1,fp8,fp8,0,1.1948586304982503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,4,1,128,1,float16,fp8,0,1.6766079266866047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,4,2,128,1,float16,fp8,0,1.6903146107991536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,4,2,128,1,fp8,fp8,0,1.2081440289815266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,4,4,128,1,float16,float16,0,0.9034026463826498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,10240,4,2,128,1,float16,float16,0,1.7100693384806316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,4,4,128,1,fp8,fp8,0,0.6018453439076742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,4,1,128,1,float16,fp8,0,0.8977013429005941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,4,1,128,1,fp8,fp8,0,0.591322660446167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,4,1,128,1,float16,float16,0,0.9032853444417318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,4,4,128,1,float16,fp8,0,0.890725294748942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,4,2,128,1,float16,float16,0,0.9118613402048746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,4,2,128,1,float16,fp8,0,0.9046613375345866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,10240,4,2,128,1,fp8,fp8,0,0.6013280153274536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,4,4,128,1,float16,float16,0,0.4337759812672933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,4,4,128,1,float16,fp8,0,0.4359519879023234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,4,4,128,1,fp8,fp8,0,0.32106133302052814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,4,1,128,1,fp8,fp8,0,0.3184426625569661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,4,1,128,1,float16,fp8,0,0.4319466749827067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,4,1,128,1,float16,float16,0,0.435263991355896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,4,2,128,1,float16,fp8,0,0.43321065107981366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,4,2,128,1,fp8,fp8,0,0.3213333288828532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,10240,4,2,128,1,float16,float16,0,0.4390559991200765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,4,4,128,1,float16,float16,0,0.24601600567499796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,4,4,128,1,float16,fp8,0,0.24522133668263754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,4,4,128,1,fp8,fp8,0,0.18639999628067017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,4,1,128,1,float16,float16,0,0.2455093264579773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,4,1,128,1,float16,fp8,0,0.24547733863194784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,4,2,128,1,float16,float16,0,0.24788800875345865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,4,1,128,1,fp8,fp8,0,0.1813653310139974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,4,2,128,1,float16,fp8,0,0.2461599906285604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,10240,4,2,128,1,fp8,fp8,0,0.18581332763036093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,4,1,128,1,float16,float16,0,2.2095093727111816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,4,1,128,1,fp8,fp8,0,1.5684746106465657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,4,1,128,1,float16,fp8,0,2.1720266342163086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,4,2,128,1,float16,float16,0,2.216656049092611
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,4,2,128,1,float16,fp8,0,2.235050678253174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,4,4,128,1,float16,float16,0,1.149722655614217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,8192,4,2,128,1,fp8,fp8,0,1.5882186889648438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,4,1,128,1,float16,float16,0,1.144917329152425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,4,1,128,1,float16,fp8,0,1.1387786865234375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,4,1,128,1,fp8,fp8,0,0.8242560227711996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,4,4,128,1,fp8,fp8,0,0.8356693585713705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,4,4,128,1,float16,fp8,0,1.1413280169169109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,4,2,128,1,float16,float16,0,1.1551146507263184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,4,2,128,1,float16,fp8,0,1.1413919925689697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,8192,4,2,128,1,fp8,fp8,0,0.8332640329996744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,4,4,128,1,float16,float16,0,0.6194666624069214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,4,4,128,1,float16,fp8,0,0.620464007059733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,4,4,128,1,fp8,fp8,0,0.4179519812266032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,4,1,128,1,fp8,fp8,0,0.40808534622192383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,4,1,128,1,float16,fp8,0,0.6182080109914144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,4,1,128,1,float16,float16,0,0.6223146518071493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,4,2,128,1,float16,float16,0,0.6248106559117635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,4,4,128,1,float16,float16,0,0.293338676293691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,4,2,128,1,float16,fp8,0,0.6210293372472128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,8192,4,2,128,1,fp8,fp8,0,0.41590933005015057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,4,1,128,1,float16,fp8,0,0.28829334179560345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,4,4,128,1,float16,fp8,0,0.29047999779383343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,4,4,128,1,fp8,fp8,0,0.22509332497914633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,4,1,128,1,float16,float16,0,0.2911840081214905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,4,1,128,1,fp8,fp8,0,0.21982399622599283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,4,2,128,1,float16,float16,0,0.2943413257598877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,4,2,128,1,float16,fp8,0,0.2905866702397664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,8192,4,2,128,1,fp8,fp8,0,0.2230506738026937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,4,4,128,1,float16,float16,0,0.1609813372294108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,4,1,128,1,float16,float16,0,0.15918933351834616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,4,1,128,1,float16,fp8,0,0.15818666418393454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,4,1,128,1,fp8,fp8,0,0.12054933110872905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,4,4,128,1,float16,fp8,0,0.16155200203259787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,4,4,128,1,fp8,fp8,0,0.1265120009581248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,4,2,128,1,float16,float16,0,0.15997333327929178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,4,2,128,1,float16,fp8,0,0.15898133317629495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,8192,4,2,128,1,fp8,fp8,0,0.12217600146929423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,4,1,128,1,float16,float16,0,1.3389546076456706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,4,1,128,1,float16,fp8,0,1.3337759971618652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,4,1,128,1,fp8,fp8,0,0.9808693726857504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,4,4,128,1,float16,float16,0,0.7089973290761312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,4,2,128,1,float16,float16,0,1.3541280428568523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,4,2,128,1,fp8,fp8,0,0.9949813683827718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,6144,4,2,128,1,float16,fp8,0,1.3262826601664226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,4,4,128,1,float16,fp8,0,0.7013440132141113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,4,1,128,1,float16,float16,0,0.7070773442586263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,4,4,128,1,fp8,fp8,0,0.5114719867706299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,4,1,128,1,float16,fp8,0,0.6940053304036459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,4,1,128,1,fp8,fp8,0,0.49141331513722736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,4,2,128,1,float16,float16,0,0.7090240319569906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,4,2,128,1,float16,fp8,0,0.7003359794616699
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,4,4,128,1,float16,float16,0,0.34650135040283203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,6144,4,2,128,1,fp8,fp8,0,0.5027786493301392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,4,4,128,1,float16,fp8,0,0.3452266852060954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,4,4,128,1,fp8,fp8,0,0.2688266634941101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,4,1,128,1,float16,fp8,0,0.3327946662902832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,4,1,128,1,fp8,fp8,0,0.25990933179855347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,4,1,128,1,float16,float16,0,0.33804798126220703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,4,2,128,1,float16,fp8,0,0.33949331442515057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,4,2,128,1,fp8,fp8,0,0.2648853262265523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,6144,4,2,128,1,float16,float16,0,0.3438346783320109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,4,4,128,1,float16,float16,0,0.19591466585795084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,4,4,128,1,float16,fp8,0,0.1941279967625936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,4,4,128,1,fp8,fp8,0,0.15361066659291586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,4,1,128,1,float16,float16,0,0.19193599621454874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,4,1,128,1,fp8,fp8,0,0.14793599645296732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,4,1,128,1,float16,fp8,0,0.18939733505249023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,4,2,128,1,float16,fp8,0,0.1925119956334432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,4,2,128,1,fp8,fp8,0,0.15130666891733804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,4,4,128,1,float16,float16,0,0.12037866314252217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,6144,4,2,128,1,float16,float16,0,0.19488000869750977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,4,4,128,1,float16,fp8,0,0.12069333593050639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,4,4,128,1,fp8,fp8,0,0.09542933106422424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,4,1,128,1,float16,fp8,0,0.11930666367212932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,4,1,128,1,float16,float16,0,0.12051733334859212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,4,1,128,1,fp8,fp8,0,0.09215466181437175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,4,2,128,1,float16,float16,0,0.12056533495585124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,4,2,128,1,fp8,fp8,0,0.09310932954152425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,6144,4,2,128,1,float16,fp8,0,0.1204853355884552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,4,1,128,1,float16,float16,0,1.3553333282470703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,4,1,128,1,fp8,fp8,0,1.0217546621958415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,4,1,128,1,float16,fp8,0,1.3090986410776775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,4,2,128,1,float16,fp8,0,1.3237173557281494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,4,2,128,1,float16,float16,0,1.358400026957194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,4,4,128,1,float16,float16,0,0.7059199810028076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,4096,4,2,128,1,fp8,fp8,0,1.0334186553955078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,4,1,128,1,float16,float16,0,0.6903039614359537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,4,1,128,1,float16,fp8,0,0.673642635345459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,4,1,128,1,fp8,fp8,0,0.5264799992243449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,4,4,128,1,float16,fp8,0,0.6977066993713379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,4,4,128,1,fp8,fp8,0,0.5491733153661092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,4,2,128,1,float16,float16,0,0.6984639962514242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,4,2,128,1,float16,fp8,0,0.684229294459025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,4096,4,2,128,1,fp8,fp8,0,0.5364053249359131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,4,4,128,1,float16,float16,0,0.3776053190231323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,4,1,128,1,float16,float16,0,0.3693173329035441
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,4,4,128,1,fp8,fp8,0,0.2812853256861369
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,4,4,128,1,float16,fp8,0,0.3713759978612264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,4,1,128,1,fp8,fp8,0,0.2669439911842346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,4,2,128,1,float16,float16,0,0.37435734272003174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,4,1,128,1,float16,fp8,0,0.363594651222229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,4,2,128,1,float16,fp8,0,0.36716798941294354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,4096,4,2,128,1,fp8,fp8,0,0.2738879919052124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,4,4,128,1,float16,float16,0,0.18522665898005167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,4,4,128,1,float16,fp8,0,0.18175999323527017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,4,1,128,1,float16,float16,0,0.17733333508173624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,4,1,128,1,fp8,fp8,0,0.14480533202489218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,4,4,128,1,fp8,fp8,0,0.15225600202878317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,4,2,128,1,float16,float16,0,0.179967999458313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,4,2,128,1,float16,fp8,0,0.17799999316533408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,4,1,128,1,float16,fp8,0,0.17519466082255045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,4096,4,2,128,1,fp8,fp8,0,0.1474666694800059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,4,4,128,1,float16,float16,0,0.10133333007494609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,4,4,128,1,float16,fp8,0,0.10044800241788228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,4,1,128,1,float16,float16,0,0.09719467163085938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,4,1,128,1,float16,fp8,0,0.09620799620946248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,4,4,128,1,fp8,fp8,0,0.08685333530108134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,4,2,128,1,float16,float16,0,0.09851200381914775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,4,1,128,1,fp8,fp8,0,0.07947200040022533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,4,2,128,1,float16,fp8,0,0.09718933701515198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,4,4,128,1,float16,float16,0,0.08476799726486206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,4096,4,2,128,1,fp8,fp8,0,0.08179733157157898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,4,4,128,1,float16,fp8,0,0.0849226713180542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,4,1,128,1,float16,float16,0,0.08532800277074178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,4,4,128,1,fp8,fp8,0,0.06760533154010773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,4,1,128,1,float16,fp8,0,0.08468266328175862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,4,2,128,1,float16,float16,0,0.08565333485603333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,4,1,128,1,fp8,fp8,0,0.0664213349421819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,4,2,128,1,float16,fp8,0,0.08516800403594971
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,4096,4,2,128,1,fp8,fp8,0,0.06741333504517873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,4,1,128,1,float16,float16,0,0.8385706742604574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,4,1,128,1,float16,fp8,0,0.8138666947682699
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,4,1,128,1,fp8,fp8,0,0.6589920123418173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,4,4,128,1,float16,float16,0,0.4529813528060913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,4,2,128,1,float16,float16,0,0.851738691329956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,4,2,128,1,float16,fp8,0,0.8239040374755859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,3072,4,2,128,1,fp8,fp8,0,0.6749707063039144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,4,1,128,1,float16,float16,0,0.4383093516031901
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,4,4,128,1,float16,fp8,0,0.440170685450236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,4,1,128,1,float16,fp8,0,0.42746134599049884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,4,1,128,1,fp8,fp8,0,0.3327573339144389
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,4,4,128,1,fp8,fp8,0,0.3590293327967326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,4,2,128,1,float16,float16,0,0.4425813357035319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,4,2,128,1,float16,fp8,0,0.4331413507461548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,4,4,128,1,float16,float16,0,0.22825600703557333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,4,4,128,1,float16,fp8,0,0.2258346676826477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,3072,4,2,128,1,fp8,fp8,0,0.3426986535390218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,4,1,128,1,float16,fp8,0,0.21055465936660767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,4,1,128,1,fp8,fp8,0,0.17748800913492838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,4,1,128,1,float16,float16,0,0.21412267287572226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,4,4,128,1,fp8,fp8,0,0.18948266903559366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,4,2,128,1,float16,float16,0,0.21913067499796549
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,4,4,128,1,float16,float16,0,0.12502400080362955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,4,4,128,1,float16,fp8,0,0.12459199627240498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,4,1,128,1,float16,float16,0,0.11926933129628499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,4,2,128,1,float16,fp8,0,0.21734933058420816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,3072,4,2,128,1,fp8,fp8,0,0.18244266510009766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,4,4,128,1,fp8,fp8,0,0.10705066720644633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,4,1,128,1,float16,fp8,0,0.11779200037320454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,4,1,128,1,fp8,fp8,0,0.09727999567985535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,4,2,128,1,float16,float16,0,0.12171733379364014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,4,2,128,1,fp8,fp8,0,0.10300800204277039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,4,4,128,1,float16,float16,0,0.07505600154399872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,4,4,128,1,float16,fp8,0,0.0751200020313263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,4,4,128,1,fp8,fp8,0,0.06307733555634816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,4,1,128,1,float16,float16,0,0.07306133210659027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,3072,4,2,128,1,float16,fp8,0,0.12079999844233195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,4,1,128,1,float16,fp8,0,0.07251733541488647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,4,1,128,1,fp8,fp8,0,0.05974400043487549
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,4,2,128,1,fp8,fp8,0,0.06066133578618368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,4,2,128,1,float16,float16,0,0.07400533556938171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,3072,4,2,128,1,float16,fp8,0,0.07352533439795177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,4,4,128,1,float16,float16,0,0.06701866785685222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,4,4,128,1,fp8,fp8,0,0.054570664962132774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,4,1,128,1,float16,float16,0,0.06824000179767609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,4,1,128,1,float16,fp8,0,0.06720533470312755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,4,4,128,1,float16,fp8,0,0.06710400183995564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,4,2,128,1,float16,float16,0,0.06834133466084798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,4,1,128,1,fp8,fp8,0,0.054245332876841225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,4,2,128,1,float16,fp8,0,0.06789333124955495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,3072,4,2,128,1,fp8,fp8,0,0.054192001620928444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,4,1,128,1,float16,float16,0,0.9204426606496176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,4,1,128,1,float16,fp8,0,0.8582346439361572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,4,1,128,1,fp8,fp8,0,0.7642772992451986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,4,4,128,1,float16,float16,0,0.5014400084813436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,4,2,128,1,float16,float16,0,0.9381866455078125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,4,2,128,1,fp8,fp8,0,0.7780373096466064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,2048,4,2,128,1,float16,fp8,0,0.8767893314361572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,4,1,128,1,float16,fp8,0,0.4453866481781006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,4,1,128,1,float16,float16,0,0.47414398193359375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,4,4,128,1,float16,fp8,0,0.46804265181223553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,4,4,128,1,fp8,fp8,0,0.4175146818161011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,4,1,128,1,fp8,fp8,0,0.39361600081125897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,4,2,128,1,float16,float16,0,0.48323198159535724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,4,2,128,1,float16,fp8,0,0.45524267355600995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,4,4,128,1,float16,fp8,0,0.25177599986394245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,2048,4,2,128,1,fp8,fp8,0,0.40326933066050213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,4,1,128,1,float16,float16,0,0.2495946685473124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,4,4,128,1,float16,float16,0,0.2653599977493286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,4,1,128,1,float16,fp8,0,0.23874666293462118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,4,4,128,1,fp8,fp8,0,0.21456533670425415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,4,1,128,1,fp8,fp8,0,0.19765865802764893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,4,4,128,1,float16,float16,0,0.1338933308919271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,4,2,128,1,float16,fp8,0,0.24333866437276205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,4,4,128,1,fp8,fp8,0,0.11737066507339478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,4,4,128,1,float16,fp8,0,0.12905066212018332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,4,2,128,1,float16,float16,0,0.2544906735420227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,4,1,128,1,float16,float16,0,0.1226026713848114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,2048,4,2,128,1,fp8,fp8,0,0.2039519945780436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,4,1,128,1,float16,fp8,0,0.11880000432332356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,4,1,128,1,fp8,fp8,0,0.10883200168609619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,4,2,128,1,fp8,fp8,0,0.1113813320795695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,4,2,128,1,float16,fp8,0,0.12102400263150533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,4,4,128,1,float16,float16,0,0.07284266750017802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,4,4,128,1,float16,fp8,0,0.07229333122571309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,2048,4,2,128,1,float16,float16,0,0.1251359979311625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,4,4,128,1,fp8,fp8,0,0.0675786683956782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,4,1,128,1,float16,float16,0,0.06750399867693584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,4,1,128,1,float16,fp8,0,0.06610133250554402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,4,1,128,1,fp8,fp8,0,0.058789332707722984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,4,2,128,1,float16,fp8,0,0.06740266581376393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,4,2,128,1,float16,float16,0,0.0687360018491745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,2048,4,2,128,1,fp8,fp8,0,0.062362665931383766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,4,4,128,1,float16,float16,0,0.05527466535568237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,4,4,128,1,float16,fp8,0,0.054917335510253906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,4,1,128,1,float16,float16,0,0.05462400118509928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,4,1,128,1,float16,fp8,0,0.05395199855168661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,4,4,128,1,fp8,fp8,0,0.046821330984433494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,4,1,128,1,fp8,fp8,0,0.0458133320013682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,4,2,128,1,float16,float16,0,0.054330666859944664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,4,2,128,1,float16,fp8,0,0.05407999952634176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,2048,4,2,128,1,fp8,fp8,0,0.04629333317279816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,4,4,128,1,float16,fp8,0,0.04964800179004669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,4,4,128,1,fp8,fp8,0,0.041850666205088295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,4,4,128,1,float16,float16,0,0.04972266654173533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,4,1,128,1,float16,float16,0,0.05081599950790405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,4,1,128,1,float16,fp8,0,0.0499893327554067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,4,2,128,1,float16,float16,0,0.050442665815353394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,4,1,128,1,fp8,fp8,0,0.04142399877309799
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,4,2,128,1,float16,fp8,0,0.05073066552480062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,2048,4,2,128,1,fp8,fp8,0,0.04189866781234741
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,4,1,128,1,float16,float16,0,0.6167946656545004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,4,1,128,1,float16,fp8,0,0.5468266805013021
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,4,1,128,1,fp8,fp8,0,0.5278933445612589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,4,4,128,1,float16,float16,0,0.3438986539840698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,4,2,128,1,float16,float16,0,0.6289653380711874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,4,2,128,1,fp8,fp8,0,0.537002682685852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1536,4,2,128,1,float16,fp8,0,0.5611466566721598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,4,1,128,1,float16,float16,0,0.32121066252390545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,4,4,128,1,float16,fp8,0,0.31328000624974567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,4,1,128,1,fp8,fp8,0,0.26734934250513714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,4,1,128,1,float16,fp8,0,0.2907680074373881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,4,2,128,1,float16,float16,0,0.3302453358968099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,4,4,128,1,fp8,fp8,0,0.3060906728108724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,4,2,128,1,float16,fp8,0,0.29841067393620807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,4,4,128,1,float16,float16,0,0.18169599771499634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,4,1,128,1,float16,float16,0,0.15763200322786966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,4,4,128,1,float16,fp8,0,0.17273600896199545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1536,4,2,128,1,fp8,fp8,0,0.27451199293136597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,4,4,128,1,fp8,fp8,0,0.1524799962838491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,4,1,128,1,float16,fp8,0,0.14967466394106546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,4,1,128,1,fp8,fp8,0,0.14221333463986716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,4,2,128,1,float16,float16,0,0.1637333333492279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,4,2,128,1,float16,fp8,0,0.1558133363723755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1536,4,2,128,1,fp8,fp8,0,0.1462613344192505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,4,4,128,1,float16,float16,0,0.09392000238100688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,4,4,128,1,float16,fp8,0,0.09010133147239685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,4,1,128,1,float16,float16,0,0.08461333314577739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,4,4,128,1,fp8,fp8,0,0.08778132994969685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,4,1,128,1,float16,fp8,0,0.08258133133252461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,4,1,128,1,fp8,fp8,0,0.07832000156243642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,4,2,128,1,float16,float16,0,0.08780266841252644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,4,2,128,1,float16,fp8,0,0.08534933129946391
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1536,4,2,128,1,fp8,fp8,0,0.08261333405971527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,4,4,128,1,float16,float16,0,0.05384000142415365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,4,4,128,1,float16,fp8,0,0.052373334765434265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,4,4,128,1,fp8,fp8,0,0.05077866713205973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,4,1,128,1,float16,float16,0,0.050399998823801674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,4,1,128,1,fp8,fp8,0,0.04505600035190582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,4,1,128,1,float16,fp8,0,0.05009066561857859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,4,2,128,1,float16,float16,0,0.050986667474110924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,4,2,128,1,float16,fp8,0,0.0505973349014918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1536,4,2,128,1,fp8,fp8,0,0.04643199841181437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,4,4,128,1,float16,float16,0,0.045093332727750145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,4,4,128,1,fp8,fp8,0,0.03941333293914795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,4,1,128,1,float16,float16,0,0.04533866544564565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,4,4,128,1,float16,fp8,0,0.045381332437197365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,4,1,128,1,float16,fp8,0,0.0444213350613912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,4,1,128,1,fp8,fp8,0,0.03888533264398575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,4,2,128,1,float16,float16,0,0.04510400195916494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,4,2,128,1,float16,fp8,0,0.04422933359940847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1536,4,2,128,1,fp8,fp8,0,0.038922667503356934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,4,4,128,1,float16,float16,0,0.041237334410349526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,4,4,128,1,fp8,fp8,0,0.03548266738653183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,4,4,128,1,float16,fp8,0,0.04165866722663244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,4,1,128,1,float16,float16,0,0.04221866528193156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,4,2,128,1,float16,float16,0,0.0417546679576238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,4,1,128,1,fp8,fp8,0,0.03532800078392029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,4,1,128,1,float16,fp8,0,0.04205333193143209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,4,2,128,1,float16,fp8,0,0.041738669077555336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1536,4,2,128,1,fp8,fp8,0,0.03594133257865906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,4,1,128,1,float16,float16,0,0.552560011545817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,4,1,128,1,float16,fp8,0,0.5482026735941569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,4,1,128,1,fp8,fp8,0,0.47175999482472736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,4,4,128,1,float16,float16,0,0.3354880015055339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,4,2,128,1,float16,float16,0,0.5781759818394979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,4,2,128,1,fp8,fp8,0,0.5010613203048706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,1024,4,2,128,1,float16,fp8,0,0.5655519962310791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,4,4,128,1,float16,fp8,0,0.32438933849334717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,4,1,128,1,float16,fp8,0,0.2879573305447896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,4,4,128,1,fp8,fp8,0,0.2891626755396525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,4,1,128,1,float16,float16,0,0.28902933994929
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,4,1,128,1,fp8,fp8,0,0.24887466430664062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,4,2,128,1,float16,float16,0,0.3016586701075236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,4,2,128,1,float16,fp8,0,0.29803733030955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,4,1,128,1,float16,float16,0,0.15808533628781637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,4,4,128,1,float16,float16,0,0.18173333009084067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,4,4,128,1,float16,fp8,0,0.17543466885884604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,4,1,128,1,float16,fp8,0,0.15732266505559286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,1024,4,2,128,1,fp8,fp8,0,0.25755200783411664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,4,4,128,1,fp8,fp8,0,0.15007999539375305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,4,1,128,1,fp8,fp8,0,0.13261866569519043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,4,2,128,1,float16,float16,0,0.16495999693870544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,4,2,128,1,float16,fp8,0,0.16290666659673056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,4,4,128,1,float16,float16,0,0.0990559955437978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,4,4,128,1,float16,fp8,0,0.09618133306503296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,1024,4,2,128,1,fp8,fp8,0,0.134853333234787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,4,1,128,1,float16,float16,0,0.08339732885360718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,4,4,128,1,fp8,fp8,0,0.08387200037638347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,4,1,128,1,float16,fp8,0,0.08220799763997395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,4,1,128,1,fp8,fp8,0,0.07249066730340321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,4,2,128,1,float16,float16,0,0.0864586631457011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,4,2,128,1,float16,fp8,0,0.08451199531555176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,1024,4,2,128,1,fp8,fp8,0,0.07614933451016744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,4,1,128,1,float16,float16,0,0.04641599953174591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,4,4,128,1,fp8,fp8,0,0.049839998284975685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,4,1,128,1,fp8,fp8,0,0.042490666111310325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,4,1,128,1,float16,fp8,0,0.04699199895064036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,4,2,128,1,float16,float16,0,0.04861866434415182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,4,4,128,1,float16,float16,0,0.05319466690222422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,4,4,128,1,float16,fp8,0,0.052373334765434265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,4,4,128,1,float16,float16,0,0.03741333385308584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,4,2,128,1,float16,fp8,0,0.047600001096725464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,1024,4,2,128,1,fp8,fp8,0,0.043663998444875084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,4,4,128,1,fp8,fp8,0,0.03253333270549774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,4,4,128,1,float16,fp8,0,0.03726933399836222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,4,1,128,1,float16,float16,0,0.03547733277082443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,4,1,128,1,fp8,fp8,0,0.03136000037193298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,4,2,128,1,float16,float16,0,0.03605866680542628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,4,2,128,1,fp8,fp8,0,0.03183999905983607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,4,1,128,1,float16,fp8,0,0.035461333890755974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,1024,4,2,128,1,float16,fp8,0,0.035536001125971474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,4,4,128,1,float16,float16,0,0.03257066756486893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,4,4,128,1,fp8,fp8,0,0.02888533224662145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,4,1,128,1,float16,fp8,0,0.032672000428040825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,4,1,128,1,float16,float16,0,0.03242666771014532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,4,4,128,1,float16,fp8,0,0.03329066683848699
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,4,2,128,1,float16,float16,0,0.03234666585922241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,4,2,128,1,float16,fp8,0,0.03249066571394602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,4,1,128,1,fp8,fp8,0,0.02811199923356374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,1024,4,2,128,1,fp8,fp8,0,0.028789333999156952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,4,4,128,1,float16,float16,0,0.030432000756263733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,4,4,128,1,float16,fp8,0,0.030853333572546642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,4,1,128,1,float16,float16,0,0.031290667752424874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,4,1,128,1,float16,fp8,0,0.0313226655125618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,4,2,128,1,float16,float16,0,0.03143999973932902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,4,4,128,1,fp8,fp8,0,0.027061333258946735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,4,2,128,1,float16,fp8,0,0.03190399954716364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,4,2,128,1,fp8,fp8,0,0.027679999669392902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,1024,4,1,128,1,fp8,fp8,0,0.0268053337931633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,4,1,128,1,float16,float16,0,0.4418133497238159
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,4,1,128,1,float16,fp8,0,0.4345173438390096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,4,1,128,1,fp8,fp8,0,0.3962399959564209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,4,2,128,1,float16,fp8,0,0.4606506824493408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,4,4,128,1,float16,float16,0,0.2808319926261902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,4,2,128,1,fp8,fp8,0,0.41965333620707196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,512,4,2,128,1,float16,float16,0,0.47063998381296795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,4,1,128,1,float16,float16,0,0.23118933041890463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,4,1,128,1,float16,fp8,0,0.22939733664194742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,4,4,128,1,float16,fp8,0,0.27136532465616864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,4,4,128,1,fp8,fp8,0,0.24784533182779947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,4,1,128,1,fp8,fp8,0,0.2082186738650004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,4,2,128,1,float16,float16,0,0.2458453377087911
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,4,2,128,1,float16,fp8,0,0.24001065889994302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,512,4,2,128,1,fp8,fp8,0,0.21615999937057495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,4,1,128,1,float16,fp8,0,0.1256586710611979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,4,1,128,1,float16,float16,0,0.12703466415405273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,4,4,128,1,float16,fp8,0,0.1469386617342631
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,4,4,128,1,float16,float16,0,0.15250133474667868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,4,4,128,1,fp8,fp8,0,0.1306613286336263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,4,1,128,1,fp8,fp8,0,0.11653866370519002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,4,2,128,1,float16,float16,0,0.13462400436401367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,4,4,128,1,float16,float16,0,0.0883733332157135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,4,4,128,1,float16,fp8,0,0.084714670976003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,4,4,128,1,fp8,fp8,0,0.07540800174077351
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,4,2,128,1,float16,fp8,0,0.1325866679350535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,512,4,2,128,1,fp8,fp8,0,0.11946133772532146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,4,1,128,1,float16,fp8,0,0.06931200126806895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,4,1,128,1,float16,float16,0,0.06917333106199901
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,4,1,128,1,fp8,fp8,0,0.06471466521422069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,4,2,128,1,fp8,fp8,0,0.06672533353169759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,4,2,128,1,float16,fp8,0,0.07171200215816498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,512,4,2,128,1,float16,float16,0,0.07366933425267537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,4,4,128,1,float16,float16,0,0.04704533517360687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,4,1,128,1,float16,float16,0,0.04021333406368891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,4,4,128,1,float16,fp8,0,0.046442667643229164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,4,1,128,1,float16,fp8,0,0.040378667414188385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,4,1,128,1,fp8,fp8,0,0.03701333453257879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,4,2,128,1,float16,float16,0,0.042208001017570496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,4,4,128,1,fp8,fp8,0,0.04580800235271454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,4,2,128,1,float16,fp8,0,0.04231466849644979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,512,4,2,128,1,fp8,fp8,0,0.03844800094763438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,4,4,128,1,float16,float16,0,0.031152000029881794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,4,4,128,1,float16,fp8,0,0.031189332405726116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,4,4,128,1,fp8,fp8,0,0.0281333327293396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,4,1,128,1,float16,float16,0,0.02869333326816559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,4,1,128,1,fp8,fp8,0,0.026816000541051228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,4,1,128,1,float16,fp8,0,0.028677334388097126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,4,2,128,1,float16,float16,0,0.02943466603755951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,4,2,128,1,fp8,fp8,0,0.027269333600997925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,512,4,2,128,1,float16,fp8,0,0.029504001140594482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,4,4,128,1,float16,fp8,0,0.02628266563018163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,4,4,128,1,float16,float16,0,0.026421333352724712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,4,4,128,1,fp8,fp8,0,0.02460266649723053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,4,1,128,1,float16,fp8,0,0.025589334468046825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,4,1,128,1,fp8,fp8,0,0.02454400062561035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,4,2,128,1,float16,float16,0,0.025749333202838898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,4,2,128,1,float16,fp8,0,0.025914666553338368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,4,1,128,1,float16,float16,0,0.025311999022960663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,512,4,2,128,1,fp8,fp8,0,0.024085332949956257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,4,4,128,1,float16,fp8,0,0.024725332856178284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,4,4,128,1,fp8,fp8,0,0.022778667509555817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,4,4,128,1,float16,float16,0,0.023914667467276256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,4,1,128,1,float16,float16,0,0.023946667710940044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,4,1,128,1,float16,fp8,0,0.02426133304834366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,4,2,128,1,float16,float16,0,0.024613333245118458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,4,1,128,1,fp8,fp8,0,0.022656001150608063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,4,2,128,1,float16,fp8,0,0.024890666206677754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,512,4,2,128,1,fp8,fp8,0,0.022885332504908245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,4,4,128,1,float16,fp8,0,0.023285334308942158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,4,4,128,1,float16,float16,0,0.02260799954334895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,4,1,128,1,float16,float16,0,0.024192000428835552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,4,1,128,1,fp8,fp8,0,0.02277333289384842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,4,4,128,1,fp8,fp8,0,0.022367998957633972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,4,2,128,1,float16,float16,0,0.023749334116776783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,4,2,128,1,float16,fp8,0,0.024522667129834492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,4,1,128,1,float16,fp8,0,0.024842667082945507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,512,4,2,128,1,fp8,fp8,0,0.021909333765506744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,4,1,128,1,float16,float16,0,0.19990400473276773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,4,1,128,1,float16,fp8,0,0.19732799132665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,4,1,128,1,fp8,fp8,0,0.18503999710083008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,4,2,128,1,float16,float16,0,0.2171786626180013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,4,2,128,1,float16,fp8,0,0.21099199851353964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,4,4,128,1,float16,float16,0,0.14178666472434998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,256,4,2,128,1,fp8,fp8,0,0.20390933752059937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,4,4,128,1,float16,fp8,0,0.13890666762987772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,4,1,128,1,float16,float16,0,0.1104906698067983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,4,1,128,1,fp8,fp8,0,0.10513066252072652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,4,1,128,1,float16,fp8,0,0.10931199789047241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,4,4,128,1,fp8,fp8,0,0.12186132868131001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,4,2,128,1,float16,float16,0,0.11867733796437581
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,4,2,128,1,float16,fp8,0,0.11691733201344807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,4,1,128,1,float16,float16,0,0.06190933287143707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,4,4,128,1,float16,float16,0,0.08406399687131245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,4,4,128,1,float16,fp8,0,0.08061866462230682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,4,4,128,1,fp8,fp8,0,0.06989333530267079
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,256,4,2,128,1,fp8,fp8,0,0.1083679993947347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,4,1,128,1,float16,fp8,0,0.06188266475995382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,4,2,128,1,float16,float16,0,0.06702400247255962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,4,2,128,1,float16,fp8,0,0.06547733147939046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,4,4,128,1,float16,float16,0,0.044031997521718345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,4,2,128,1,fp8,fp8,0,0.06329600016276042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,256,4,1,128,1,fp8,fp8,0,0.0599839985370636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,4,4,128,1,float16,fp8,0,0.04264000058174133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,4,1,128,1,float16,float16,0,0.03809066613515218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,4,2,128,1,float16,float16,0,0.04004266609748205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,4,1,128,1,float16,fp8,0,0.03839466720819473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,4,4,128,1,fp8,fp8,0,0.04155199974775314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,4,1,128,1,fp8,fp8,0,0.034490667283535004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,4,2,128,1,fp8,fp8,0,0.03637866675853729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,256,4,2,128,1,float16,fp8,0,0.03933866570393244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,4,4,128,1,float16,float16,0,0.02900800108909607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,4,4,128,1,float16,fp8,0,0.028463999430338543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,4,4,128,1,fp8,fp8,0,0.025616000096003216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,4,1,128,1,float16,float16,0,0.026181332767009735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,4,1,128,1,float16,fp8,0,0.0269813338915507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,4,2,128,1,float16,float16,0,0.027322667340437572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,4,1,128,1,fp8,fp8,0,0.02362666775782903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,4,2,128,1,float16,fp8,0,0.02738133321205775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,4,4,128,1,float16,float16,0,0.02313599983851115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,256,4,2,128,1,fp8,fp8,0,0.0240639994541804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,4,2,128,1,float16,float16,0,0.02266666789849599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,4,4,128,1,fp8,fp8,0,0.02139200021823247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,4,1,128,1,float16,fp8,0,0.02221333235502243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,4,1,128,1,fp8,fp8,0,0.020869334538777668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,4,1,128,1,float16,float16,0,0.02204799900452296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,4,4,128,1,float16,fp8,0,0.023408000667889912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,4,2,128,1,fp8,fp8,0,0.021210665504137676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,4,4,128,1,float16,float16,0,0.02049066623051961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,4,4,128,1,float16,fp8,0,0.021903999149799347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,4,1,128,1,fp8,fp8,0,0.01956266661485036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,4,4,128,1,fp8,fp8,0,0.019882666567961376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,4,1,128,1,float16,float16,0,0.021045332153638203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,256,4,2,128,1,float16,fp8,0,0.022453332940737408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,4,2,128,1,fp8,fp8,0,0.02019199977318446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,4,1,128,1,float16,fp8,0,0.02103466788927714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,4,2,128,1,float16,float16,0,0.02094399929046631
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,256,4,2,128,1,float16,fp8,0,0.02189333240191142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,4,4,128,1,float16,float16,0,0.020634666085243225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,4,1,128,1,float16,float16,0,0.020495999604463577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,4,4,128,1,fp8,fp8,0,0.019109333554903667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,4,4,128,1,float16,fp8,0,0.021274665991465252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,4,1,128,1,fp8,fp8,0,0.01939733326435089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,4,2,128,1,float16,float16,0,0.02077866718173027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,4,2,128,1,float16,fp8,0,0.02040533348917961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,4,2,128,1,fp8,fp8,0,0.01923199991385142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,256,4,1,128,1,float16,fp8,0,0.020655999581019085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,4,4,128,1,float16,float16,0,0.019760000209013622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,4,4,128,1,float16,fp8,0,0.020474666108687718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,4,1,128,1,fp8,fp8,0,0.019007999449968338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,4,4,128,1,fp8,fp8,0,0.01829333355029424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,4,1,128,1,float16,float16,0,0.020117333779732387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,4,1,128,1,float16,fp8,0,0.020282667130231857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,4,2,128,1,float16,fp8,0,0.020645332833131153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,4,2,128,1,fp8,fp8,0,0.01911466692884763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,256,4,2,128,1,float16,float16,0,0.019472000499566395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,4,1,128,1,float16,float16,0,0.11174933115641277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,4,1,128,1,fp8,fp8,0,0.10253866513570149
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,4,1,128,1,float16,fp8,0,0.1074773371219635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,4,2,128,1,float16,float16,0,0.11736533045768738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,4,2,128,1,float16,fp8,0,0.1153706709543864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,4,4,128,1,float16,float16,0,0.07997333506743114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,128,4,2,128,1,fp8,fp8,0,0.10850666960080464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,4,1,128,1,float16,float16,0,0.05919999877611796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,4,1,128,1,float16,fp8,0,0.059258664647738137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,4,4,128,1,fp8,fp8,0,0.06902400155862172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,4,4,128,1,float16,fp8,0,0.07681066791216533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,4,1,128,1,fp8,fp8,0,0.0601440022389094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,4,2,128,1,float16,fp8,0,0.06345599889755249
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,4,2,128,1,fp8,fp8,0,0.06215466558933258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,4,1,128,1,float16,float16,0,0.03643733263015747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,4,4,128,1,float16,float16,0,0.04256533086299896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,4,4,128,1,float16,fp8,0,0.04055466751257578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,4,4,128,1,fp8,fp8,0,0.04071466624736786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,128,4,2,128,1,float16,float16,0,0.06496533254782359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,4,4,128,1,float16,float16,0,0.027669332921504974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,4,1,128,1,fp8,fp8,0,0.03392533212900162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,4,1,128,1,float16,fp8,0,0.036506667733192444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,4,2,128,1,fp8,fp8,0,0.035589332381884255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,4,2,128,1,float16,fp8,0,0.0377813329299291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,128,4,2,128,1,float16,float16,0,0.03799466788768768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,4,4,128,1,float16,fp8,0,0.02736533433198929
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,4,4,128,1,fp8,fp8,0,0.025248001019159954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,4,1,128,1,float16,float16,0,0.025722667574882507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,4,1,128,1,fp8,fp8,0,0.02387733260790507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,4,2,128,1,float16,float16,0,0.026421333352724712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,4,2,128,1,float16,fp8,0,0.025770666698614757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,4,1,128,1,float16,fp8,0,0.025455998877684276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,4,4,128,1,float16,float16,0,0.02160000056028366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,4,4,128,1,float16,fp8,0,0.021151999632517498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,128,4,2,128,1,fp8,fp8,0,0.024314666787783306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,4,4,128,1,fp8,fp8,0,0.021386665602525074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,4,1,128,1,float16,float16,0,0.021141332884629566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,4,1,128,1,float16,fp8,0,0.021002667645613354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,4,2,128,1,float16,float16,0,0.021349333226680756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,4,1,128,1,fp8,fp8,0,0.020879998803138733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,4,4,128,1,float16,float16,0,0.019861333072185516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,4,2,128,1,float16,fp8,0,0.020949333906173706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,128,4,2,128,1,fp8,fp8,0,0.02085866779088974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,4,4,128,1,fp8,fp8,0,0.019589333484570186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,4,4,128,1,float16,fp8,0,0.02027200038234393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,4,1,128,1,float16,float16,0,0.019839999576409657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,4,1,128,1,float16,fp8,0,0.020175999651352566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,4,2,128,1,float16,float16,0,0.019727999965349834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,4,4,128,1,float16,float16,0,0.018559999763965607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,4,2,128,1,fp8,fp8,0,0.019925333559513092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,4,2,128,1,float16,fp8,0,0.01989866668979327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,128,4,1,128,1,fp8,fp8,0,0.01970133309563001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,4,4,128,1,float16,fp8,0,0.01918399954835574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,4,1,128,1,float16,float16,0,0.01897066707412402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,4,4,128,1,fp8,fp8,0,0.01931200052301089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,4,1,128,1,float16,fp8,0,0.019215999792019527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,4,1,128,1,fp8,fp8,0,0.019205333044131596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,4,2,128,1,float16,float16,0,0.01868266612291336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,4,2,128,1,fp8,fp8,0,0.019258666783571243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,128,4,2,128,1,float16,fp8,0,0.019519999623298645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,4,4,128,1,float16,float16,0,0.018042666216691334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,4,4,128,1,fp8,fp8,0,0.01846933364868164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,4,1,128,1,float16,float16,0,0.01903466631968816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,4,4,128,1,float16,fp8,0,0.01929066702723503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,4,1,128,1,fp8,fp8,0,0.01878400022784869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,4,1,128,1,float16,fp8,0,0.018789333601792652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,4,2,128,1,float16,float16,0,0.019023999571800232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,4,2,128,1,float16,fp8,0,0.018789333601792652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,128,4,2,128,1,fp8,fp8,0,0.018570666511853535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,4,4,128,1,float16,float16,0,0.018383999665578205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,4,4,128,1,fp8,fp8,0,0.018464000274737675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,4,1,128,1,float16,float16,0,0.01830400029818217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,4,1,128,1,float16,fp8,0,0.01932799940307935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,4,4,128,1,float16,fp8,0,0.018901333212852478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,4,1,128,1,fp8,fp8,0,0.01876266673207283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,4,2,128,1,float16,float16,0,0.018592000007629395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,4,2,128,1,fp8,fp8,0,0.019029332945744198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,128,4,2,128,1,float16,fp8,0,0.019199999670187633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,4,1,128,1,float16,fp8,0,0.06580266853173573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,4,1,128,1,float16,float16,0,0.06588799754778545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,4,2,128,1,float16,float16,0,0.0726986676454544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,4,2,128,1,fp8,fp8,0,0.0767626663049062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,4,4,128,1,float16,float16,0,0.043882668018341064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,4,1,128,1,fp8,fp8,0,0.07414933542410533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,64,4,2,128,1,float16,fp8,0,0.07077333331108093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,4,4,128,1,float16,fp8,0,0.041706666350364685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,4,1,128,1,float16,float16,0,0.03921066721280416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,4,1,128,1,float16,fp8,0,0.03949866692225138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,4,1,128,1,fp8,fp8,0,0.04154666761557261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,4,4,128,1,fp8,fp8,0,0.049440001447995506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,4,2,128,1,float16,float16,0,0.04114133367935816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,4,2,128,1,float16,fp8,0,0.04038399954636892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,64,4,2,128,1,fp8,fp8,0,0.04456000030040741
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,4,4,128,1,float16,fp8,0,0.028031999866167705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,4,4,128,1,float16,float16,0,0.02739199995994568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,4,4,128,1,fp8,fp8,0,0.028746667007605236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,4,1,128,1,fp8,fp8,0,0.027930667002995808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,4,1,128,1,float16,float16,0,0.02681066592534383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,4,1,128,1,float16,fp8,0,0.02734400083621343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,4,2,128,1,float16,float16,0,0.027402666707833607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,4,2,128,1,float16,fp8,0,0.02775999903678894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,64,4,2,128,1,fp8,fp8,0,0.027744000156720478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,4,4,128,1,float16,float16,0,0.02143466720978419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,4,4,128,1,float16,fp8,0,0.021327999730904896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,4,1,128,1,fp8,fp8,0,0.02102400114138921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,4,1,128,1,float16,fp8,0,0.020693333198626835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,4,4,128,1,fp8,fp8,0,0.02162133405605952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,4,2,128,1,float16,fp8,0,0.02088533341884613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,4,1,128,1,float16,float16,0,0.020560000091791153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,4,2,128,1,float16,float16,0,0.020842666427294414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,64,4,2,128,1,fp8,fp8,0,0.02109333376089732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,4,4,128,1,float16,float16,0,0.01836799954374631
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,4,4,128,1,float16,fp8,0,0.019013332823912304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,4,1,128,1,float16,float16,0,0.018373332917690277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,4,1,128,1,fp8,fp8,0,0.019589333484570186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,4,1,128,1,float16,fp8,0,0.01850133389234543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,4,2,128,1,float16,float16,0,0.0183146670460701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,4,2,128,1,float16,fp8,0,0.019098666807015736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,4,4,128,1,fp8,fp8,0,0.01945066700379054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,4,4,128,1,float16,float16,0,0.017935999979575474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,64,4,2,128,1,fp8,fp8,0,0.020154666155576706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,4,4,128,1,float16,fp8,0,0.018112000077962875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,4,1,128,1,float16,float16,0,0.01793066660563151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,4,4,128,1,fp8,fp8,0,0.01897066707412402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,4,2,128,1,float16,float16,0,0.018138666947682697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,4,1,128,1,float16,fp8,0,0.018266666680574417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,4,2,128,1,float16,fp8,0,0.018613333503405254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,4,1,128,1,fp8,fp8,0,0.019130667050679524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,64,4,2,128,1,fp8,fp8,0,0.019098666807015736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,4,4,128,1,fp8,fp8,0,0.018730666488409042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,4,4,128,1,float16,float16,0,0.016949333250522614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,4,1,128,1,float16,float16,0,0.01766933376590411
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,4,4,128,1,float16,fp8,0,0.017594666530688603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,4,1,128,1,float16,fp8,0,0.01817600056529045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,4,2,128,1,float16,fp8,0,0.017973333597183228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,4,2,128,1,float16,float16,0,0.01748266691962878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,4,2,128,1,fp8,fp8,0,0.018816000471512478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,64,4,1,128,1,fp8,fp8,0,0.018778666853904724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,4,4,128,1,float16,float16,0,0.016906666258970898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,4,1,128,1,float16,float16,0,0.016842667013406754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,4,1,128,1,float16,fp8,0,0.01773333301146825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,4,4,128,1,fp8,fp8,0,0.0182239996890227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,4,1,128,1,fp8,fp8,0,0.018565333137909572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,4,4,128,1,float16,fp8,0,0.0174346665541331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,4,2,128,1,fp8,fp8,0,0.018672000616788864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,4,2,128,1,float16,fp8,0,0.017903999735911686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,64,4,2,128,1,float16,float16,0,0.016704000532627106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,4,4,128,1,float16,float16,0,0.01653333380818367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,4,4,128,1,float16,fp8,0,0.017221332838137943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,4,4,128,1,fp8,fp8,0,0.01823466643691063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,4,1,128,1,float16,fp8,0,0.017594666530688603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,4,1,128,1,fp8,fp8,0,0.01870399961868922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,4,2,128,1,float16,fp8,0,0.017221332838137943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,4,1,128,1,float16,float16,0,0.01855466639002164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,4,2,128,1,fp8,fp8,0,0.0183146670460701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,4,1,128,1,float16,float16,0,0.04653333127498627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,64,4,2,128,1,float16,float16,0,0.016682667036851246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,4,1,128,1,float16,fp8,0,0.04674133161703745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,4,1,128,1,fp8,fp8,0,0.058042665322621666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,4,2,128,1,float16,float16,0,0.04756799836953481
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,4,2,128,1,fp8,fp8,0,0.060378665725390114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,4,4,128,1,float16,float16,0,0.030847998956839245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,4,4,128,1,fp8,fp8,0,0.03719466676314672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,32,4,2,128,1,float16,fp8,0,0.04757866760094961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,4,4,128,1,float16,fp8,0,0.03068266560633977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,4,1,128,1,float16,float16,0,0.03086400032043457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,4,1,128,1,float16,fp8,0,0.030576000610987347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,4,1,128,1,fp8,fp8,0,0.0365226666132609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,4,2,128,1,float16,float16,0,0.030453334252039593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,4,2,128,1,float16,fp8,0,0.03133333226044973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,4,4,128,1,float16,fp8,0,0.022592000663280487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,4,4,128,1,float16,float16,0,0.02239999920129776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,4,4,128,1,fp8,fp8,0,0.025066666305065155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,32,4,2,128,1,fp8,fp8,0,0.037402667105197906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,4,1,128,1,float16,fp8,0,0.02204799900452296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,4,1,128,1,fp8,fp8,0,0.02532800038655599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,4,1,128,1,float16,float16,0,0.022357332209746044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,4,2,128,1,float16,float16,0,0.02236266682545344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,4,2,128,1,float16,fp8,0,0.022245332598686218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,32,4,2,128,1,fp8,fp8,0,0.025072000920772552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,4,4,128,1,float16,fp8,0,0.018624000251293182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,4,4,128,1,fp8,fp8,0,0.019989332805077236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,4,4,128,1,float16,float16,0,0.01844266677896182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,4,1,128,1,float16,float16,0,0.02000533292690913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,4,2,128,1,float16,float16,0,0.018058666338523228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,4,1,128,1,float16,fp8,0,0.01811733345190684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,4,2,128,1,float16,fp8,0,0.01841066653529803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,4,2,128,1,fp8,fp8,0,0.01966933285196622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,32,4,1,128,1,fp8,fp8,0,0.01974933346112569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,4,4,128,1,float16,fp8,0,0.017722666263580322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,4,4,128,1,fp8,fp8,0,0.01877333347996076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,4,4,128,1,float16,float16,0,0.016858667135238647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,4,1,128,1,float16,float16,0,0.01740266631046931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,4,1,128,1,fp8,fp8,0,0.01870399961868922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,4,1,128,1,float16,fp8,0,0.017898666361967724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,4,2,128,1,float16,fp8,0,0.017711999515692394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,4,2,128,1,float16,float16,0,0.01725333308180173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,32,4,2,128,1,fp8,fp8,0,0.01934933289885521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,4,4,128,1,fp8,fp8,0,0.018474667022625606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,4,4,128,1,float16,fp8,0,0.01684800038735072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,4,1,128,1,float16,float16,0,0.01704000060757001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,4,1,128,1,float16,fp8,0,0.017221332838137943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,4,4,128,1,float16,float16,0,0.01647466669480006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,4,2,128,1,float16,float16,0,0.017082666357358296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,4,2,128,1,float16,fp8,0,0.017375999440749485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,4,2,128,1,fp8,fp8,0,0.018816000471512478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,32,4,1,128,1,fp8,fp8,0,0.01882133384545644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,4,4,128,1,float16,float16,0,0.016234666109085083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,4,4,128,1,float16,fp8,0,0.016447999825080235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,4,1,128,1,float16,fp8,0,0.017269333203633625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,4,1,128,1,float16,float16,0,0.01640533283352852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,4,1,128,1,fp8,fp8,0,0.018672000616788864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,4,4,128,1,fp8,fp8,0,0.018506667266289394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,4,2,128,1,float16,fp8,0,0.017477333545684814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,4,2,128,1,fp8,fp8,0,0.018794666975736618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,4,4,128,1,float16,float16,0,0.016095999628305435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,32,4,2,128,1,float16,float16,0,0.01664000004529953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,4,4,128,1,float16,fp8,0,0.01595199977358182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,4,4,128,1,fp8,fp8,0,0.018053332964579265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,4,1,128,1,float16,fp8,0,0.016666666915019352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,4,2,128,1,float16,float16,0,0.016885332763195038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,4,1,128,1,fp8,fp8,0,0.01838933303952217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,4,2,128,1,fp8,fp8,0,0.01852799952030182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,4,1,128,1,float16,float16,0,0.016384000579516094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,4,4,128,1,float16,float16,0,0.01621866722901662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,4,4,128,1,fp8,fp8,0,0.018687999496857326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,32,4,2,128,1,float16,fp8,0,0.016314666718244553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,4,4,128,1,float16,fp8,0,0.016063999384641647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,4,1,128,1,float16,float16,0,0.016821333517630894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,4,1,128,1,float16,fp8,0,0.016789333273967106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,4,2,128,1,float16,fp8,0,0.017269333203633625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,4,2,128,1,float16,float16,0,0.01642666632930438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,4,1,128,1,fp8,fp8,0,0.018911999960740406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,4,1,128,1,float16,float16,0,0.041562666495641075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,32,4,2,128,1,fp8,fp8,0,0.021573332448800404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,4,1,128,1,float16,fp8,0,0.04222933451334635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,4,1,128,1,fp8,fp8,0,0.052202666799227394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,4,2,128,1,float16,float16,0,0.04204266766707102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,4,2,128,1,float16,fp8,0,0.04223999877770742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,4,4,128,1,float16,float16,0,0.027189334233601887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,4,4,128,1,fp8,fp8,0,0.033999999364217125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,4,1,128,1,float16,float16,0,0.028138667345046997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,256,16,4,2,128,1,fp8,fp8,0,0.053445334235827126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,4,1,128,1,float16,fp8,0,0.02775466690460841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,4,4,128,1,float16,fp8,0,0.02716800073782603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,4,1,128,1,fp8,fp8,0,0.03366400053103765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,4,2,128,1,float16,float16,0,0.028021333118279774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,4,2,128,1,float16,fp8,0,0.028431999186674755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,4,4,128,1,float16,fp8,0,0.020661332954963047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,128,16,4,2,128,1,fp8,fp8,0,0.03370666752258936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,4,4,128,1,float16,float16,0,0.020655999581019085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,4,4,128,1,fp8,fp8,0,0.023402666052182514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,4,1,128,1,float16,float16,0,0.020997333029905956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,4,1,128,1,float16,fp8,0,0.021130666136741638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,4,2,128,1,float16,float16,0,0.020879998803138733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,4,1,128,1,fp8,fp8,0,0.023951999843120575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,4,2,128,1,float16,fp8,0,0.02142400046189626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,64,16,4,2,128,1,fp8,fp8,0,0.02441066751877467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,4,4,128,1,float16,float16,0,0.017573333034912746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,4,4,128,1,float16,fp8,0,0.018016000588734944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,4,4,128,1,fp8,fp8,0,0.019194666296243668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,4,1,128,1,float16,float16,0,0.01748266691962878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,4,1,128,1,float16,fp8,0,0.01783466711640358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,4,1,128,1,fp8,fp8,0,0.01931200052301089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,4,2,128,1,float16,float16,0,0.01756799966096878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,4,4,128,1,float16,float16,0,0.016730666160583496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,4,4,128,1,float16,fp8,0,0.017093333105246227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,4,2,128,1,float16,fp8,0,0.017802666872739792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,32,16,4,2,128,1,fp8,fp8,0,0.019610666980346043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,4,4,128,1,fp8,fp8,0,0.018405333161354065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,4,1,128,1,float16,float16,0,0.016730666160583496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,4,1,128,1,float16,fp8,0,0.017535999417304993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,4,2,128,1,float16,float16,0,0.017157333592573803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,4,1,128,1,fp8,fp8,0,0.01905599981546402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,4,2,128,1,float16,fp8,0,0.01730666682124138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,4,4,128,1,float16,fp8,0,0.01653333380818367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,4,4,128,1,float16,float16,0,0.01658133293191592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,4,1,128,1,float16,float16,0,0.01728533332546552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,16,16,4,2,128,1,fp8,fp8,0,0.018960000326236088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,4,1,128,1,float16,fp8,0,0.01720533271630605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,4,1,128,1,fp8,fp8,0,0.01882133384545644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,4,4,128,1,fp8,fp8,0,0.01820266619324684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,4,2,128,1,fp8,fp8,0,0.018709332992633183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,4,2,128,1,float16,fp8,0,0.01720533271630605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,8,16,4,2,128,1,float16,float16,0,0.01684800038735072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,4,4,128,1,fp8,fp8,0,0.0182239996890227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,4,4,128,1,float16,fp8,0,0.016314666718244553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,4,1,128,1,float16,fp8,0,0.01693333312869072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,4,1,128,1,float16,float16,0,0.016607999801635742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,4,1,128,1,fp8,fp8,0,0.018725333114465077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,4,2,128,1,float16,float16,0,0.01664000004529953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,4,4,128,1,float16,float16,0,0.0162773331006368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,4,4,128,1,float16,float16,0,0.01571200042963028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,4,2,128,1,fp8,fp8,0,0.018373332917690277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,4,16,4,2,128,1,float16,fp8,0,0.016751999656359356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,4,4,128,1,float16,fp8,0,0.016352000335852306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,4,4,128,1,fp8,fp8,0,0.01825599993268649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,4,1,128,1,float16,float16,0,0.01621333385507266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,4,1,128,1,fp8,fp8,0,0.01817600056529045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,4,2,128,1,float16,float16,0,0.016149333367745083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,4,2,128,1,float16,fp8,0,0.01632000009218852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,4,4,128,1,fp8,fp8,0,0.017957333475351334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,4,1,128,1,float16,fp8,0,0.01685333376129468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,2,16,4,2,128,1,fp8,fp8,0,0.01850133389234543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,4,4,128,1,float16,float16,0,0.016261332978804905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,4,4,128,1,float16,fp8,0,0.015978666643301647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,4,1,128,1,fp8,fp8,0,0.01848000039656957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,4,1,128,1,float16,float16,0,0.016271999726692837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,4,1,128,1,float16,fp8,0,0.016693333784739178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,4,2,128,1,float16,fp8,0,0.016997333616018295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,4,2,128,1,float16,float16,0,0.01642666632930438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,context_attention,torch_flow,1,16,4,2,128,1,fp8,fp8,0,0.01868266612291336
